validator.cpp

Go to the documentation of this file.

#include <ISM/utility/TableHelper.hpp>
#include <ISM/common_type/RecordedPattern.hpp>
#include <ISM/recognizer/Recognizer.hpp>
#include <boost/program_options.hpp>
#include <string>
#include <set>
#include <stack>
#include <utility>

using namespace ISM;
using namespace std;
namespace po = boost::program_options;

bool detectGeneric = false;
int onlyN = -1;
int maxRuns = -1;
int runs = 0;

void validatePattern(RecordedPatternPtr pattern, RecognizerPtr recognizer) {
    if (maxRuns >= 0 && runs >= maxRuns) {
        return;
    }
    cout << "validate pattern " << pattern->name << endl;
    int idx = 0;
    int setCount = 0;
    int objectCount = 0;
    int identifySum = 0;
    double confidenceSum = 0;
    double thresholdConfidence = 0.8;
    for (ObjectSetPtr& os : pattern->objectSets) {
        if (onlyN >= 0 && onlyN != idx) {
            idx++;
            continue;
        }

        if (maxRuns >= 0 && runs >= maxRuns) {
            break;
        }

        set<pair<string, string> > mappedTypes;

        if (detectGeneric)
        {
            for (ObjectPtr& obj : os->objects) {
                mappedTypes.insert(make_pair(obj->type, obj->observedId));
                objectCount++;
                obj->type = "";
                obj->observedId = "";
            }
        }
        setCount++;
        runs++;

        vector<RecognitionResultPtr> results = recognizer->recognizePattern(os);
        bool found = false;
        for (RecognitionResultPtr& result : results) {
      if (result->patternName == pattern->name) {
                vector<RecognitionResultPtr> all;
                all.push_back(result);
                stack<RecognitionResultPtr> subPatterns;

                for (RecognitionResultPtr& subPattern : result->subPatterns) {
                    subPatterns.push(subPattern);
                }

                while (subPatterns.size() > 0) {
                    RecognitionResultPtr sp = subPatterns.top();
                    subPatterns.pop();
                    all.push_back(sp);
                    for (RecognitionResultPtr& subPattern : sp->subPatterns) {
                        subPatterns.push(subPattern);
                    }
                }

                for (RecognitionResultPtr& pattern : all) {
          for (ObjectPtr& obj : pattern->recognizedSet->objects) {
            set<pair<string, string> >::iterator match = mappedTypes.find(make_pair(obj->type, obj->observedId));
                    if (match != mappedTypes.end()) {
                      mappedTypes.erase(match);
                      identifySum++;
                    }
                  }
                }

                found = true;
                confidenceSum += result->confidence;
                break;
            }
        }
        cout << (found ? "." : ",");
        cout.flush();
        idx++;
        if ((onlyN >= 0 && onlyN - 1 == idx)) {
            break;
        }
    }
    cout << endl;

    double meanConfidence = confidenceSum / (double) setCount;
    cout << (meanConfidence >= thresholdConfidence ? "SUCCESS" : "FAILURE") << " mean confidence for pattern '"
            << pattern->name << "' is " << meanConfidence << endl;
    if (detectGeneric) {
        double identifyRate = (double) identifySum / (double) objectCount;
        cout << "Identification rate: " << identifyRate << endl;
    }
}

int main(int argc, char** argv) {
    double bin_size = 0.005;

    po::options_description desc("Allowed options");
    desc.add_options()("help,h", "produce help message")("database-file,d",
            po::value<string>()->default_value("record.sqlite"), "database file to use")("generic-mode,g",
            po::bool_switch(&detectGeneric),
            "test object type inference by removing object type and id from recognition input")("bin_size,s",
            po::value<double>(&bin_size)->default_value(0.0001), "recognizer bin_size")("pattern-name,p",
            po::value<vector<string> >(), "patters to validate instead of all")("onlyN,o",
            po::value<int>(&onlyN)->default_value(-1), "only the n'th set")("maxRuns,m",
            po::value<int>(&maxRuns)->default_value(-1), "test a maximum of m sets");

    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, desc), vm);
    po::notify(vm);

    if (vm.count("help")) {
        cout << desc << "\n";
        return 1;
    }

    TableHelper t(vm["database-file"].as<string>());
    RecognizerPtr r(new Recognizer(vm["database-file"].as<string>(), bin_size, 10, false));
    if (vm.count("pattern-name")) {
        vector<string> patternNames = vm["pattern-name"].as<vector<string> >();
        for (string& name : patternNames) {
            RecordedPatternPtr pattern = t.getRecordedPattern(name);
            if (!pattern) {
                cout << "Pattern " << name << " not found!";
                continue;
            }
            validatePattern(pattern, r);
        }
    } else {
        vector<string> patternNames = t.getRecordedPatternNames();
        for (string& name : patternNames) {
            RecordedPatternPtr pattern = t.getRecordedPattern(name);
            if (!pattern) {
                cout << "Pattern " << name << " not found!";
                continue;
            }
            validatePattern(pattern, r);
        }
    }
}