core.hh

Go to the documentation of this file.

#ifndef _RLCORE_H_
#define _RLCORE_H_

#include "Random.h"
#include <vector>
#include <map>


// types of models
#define RMAX        0
#define TABULAR     0
#define SLF         1
#define C45TREE     2
#define SINGLETREE  3
#define SVM         4
#define STUMP       5
#define M5MULTI     6
#define M5SINGLE    7
#define M5ALLMULTI  8
#define M5ALLSINGLE 9 
#define LSTMULTI    10
#define LSTSINGLE   11
#define ALLM5TYPES  12
#define GPREGRESS   13
#define GPTREE      14

const std::string modelNames[] = {
  "Tabular",
  "SLF",
  "C4.5 Tree",
  "Single Tree",
  "SVM",
  "Stump",
  "M5 Tree",
  "M5 Tree",
  "M5 Tree",
  "M5 Tree",
  "LS Tree",
  "LS Tree",
  "M5 Combo",
  "GP Regression",
  "GP Tree"
};

// types of model combos
#define AVERAGE        1
#define WEIGHTAVG      2
#define BEST           3
#define SEPARATE       4 // sep model for planning, and forest for uncertainty

const std::string comboNames[] = {
  "Average",
  "Weighted Average",
  "Best",
  "Separate"
};

// types of exploration
#define EXPLORE_UNKNOWN    0
#define TWO_MODE           1
#define TWO_MODE_PLUS_R    2
#define CONTINUOUS_BONUS   3
#define THRESHOLD_BONUS    4
#define CONTINUOUS_BONUS_R 5
#define THRESHOLD_BONUS_R  6
#define NO_EXPLORE         7
#define GREEDY             7
#define EPSILONGREEDY      8
#define VISITS_CONF        9
#define UNVISITED_BONUS    11
#define UNVISITED_ACT_BONUS 13
#define DIFF_AND_VISIT_BONUS 16
#define NOVEL_STATE_BONUS    18
#define DIFF_AND_NOVEL_BONUS 19

const std::string exploreNames[] = {
  "Explore Unknowns",
  "Two Modes",
  "Two Models +R",
  "Continuous Bonus",
  "Threshold Bonus",
  "Continuous Bonus +R",
  "Threshold Bonus +R",
  "Greedy",
  "Epsilon-Greedy",
  "Visits Confidence",
  "Type 10",
  "Unvisited State Bonus",
  "Type 12", 
  "Unvisited Action Bonus",
  "Type 14",
  "Type 15",
  "Model Diff & Visit Bonus",
  "Type 17",
  "FeatDist Bonus",
  "Model Diff & FeatDist Bonus"
};

// types of planners
#define VALUE_ITERATION    0
#define POLICY_ITERATION   1
#define PRI_SWEEPING       2
#define UCT                3
#define ET_UCT             4
#define ET_UCT_WITH_ENV    5
#define SWEEPING_UCT_HYBRID 6
#define CMAC_PLANNER       7
#define NN_PLANNER         8
#define MOD_PRI_SWEEPING   9
#define ET_UCT_L1          10
#define UCT_WITH_L         11
#define UCT_WITH_ENV       12
#define PARALLEL_ET_UCT    13
#define ET_UCT_ACTUAL      14
#define ET_UCT_CORNERS     15
#define PAR_ETUCT_ACTUAL   16
#define PAR_ETUCT_CORNERS  17
#define POMDP_ETUCT        18
#define POMDP_PAR_ETUCT    19
#define MBS_VI             20

const std::string plannerNames[] = {
  "Value Iteration",
  "Policy Iteration",
  "Prioritized Sweeping",
  "UCT",
  "UCT",
  "UCT",
  "Sweeping UCT Hybrid",
  "CMACs",
  "NN",
  "Mod. Pri Sweeping",
  "UCT L=1",
  "UCT L",
  "UCT Env",
  "Parallel UCT",
  "Real-Valued UCT",
  "Corner UCT",
  "Parallel Real-Valued UCT",
  "Parallel Corner UCT",
  "Delayed UCT",
  "Parallel Delayed UCT",
  "Model Based Simulation - VI"
};
  


#define EPSILON   1e-5

struct experience {
  std::vector<float> s;
  int act;
  float reward;
  std::vector<float> next;
  bool terminal;
};

struct modelPair {
  std::vector<float> in;
  std::vector<float> out;
};

struct classPair {
  std::vector<float> in;
  float out;
};

class Environment {
public:
  virtual const std::vector<float> &sensation() const = 0;

  virtual float apply(int action) = 0;

  virtual bool terminal() const = 0;

  virtual void reset() = 0;

  virtual int getNumActions() = 0;

  virtual void getMinMaxFeatures(std::vector<float> *minFeat,
                                 std::vector<float> *maxFeat) = 0;

  virtual void getMinMaxReward(float *minR, float *maxR) = 0;

  virtual bool isEpisodic(){ return true; };

  virtual std::vector<experience> getSeedings()
  {
    std::vector<experience> e;
    return e;
  } ;

  virtual void setSensation(std::vector<float> s){};

  virtual ~Environment() {};

};

class Agent {
public:
  virtual int first_action(const std::vector<float> &s) = 0;

  virtual int next_action(float r, const std::vector<float> &s) = 0;

  virtual void last_action(float r) = 0;

  virtual void setDebug(bool d) = 0;

  virtual void seedExp(std::vector<experience> seeds) {};

  virtual void savePolicy(const char* filename) {};

  virtual ~Agent() {};
};

class Model {
public:
  virtual bool trainInstances(std::vector<modelPair> &instances) = 0;

  virtual bool trainInstance(modelPair &instance) = 0;

  virtual std::vector<float> testInstance(const std::vector<float> &in) = 0;

  virtual ~Model() {};
};

class Classifier {
public:
  virtual bool trainInstances(std::vector<classPair> &instances) = 0;

  virtual bool trainInstance(classPair &instance) = 0;

  virtual void testInstance(const std::vector<float> &in, std::map<float, float>* retval) = 0;

  virtual float getConf(const std::vector<float> &in) = 0;

  virtual Classifier* getCopy() = 0;

  virtual ~Classifier() {};
};

struct StateActionInfo {
  bool known;
  float reward;
  float termProb;
  int frameUpdated;

  // map from outcome state to probability
  std::map< std::vector<float> , float> transitionProbs;

  StateActionInfo(){
    known = false;
    reward = 0.0;
    termProb = 0.0;
    frameUpdated = -1;
  };
};


class MDPModel {
public:
  virtual bool updateWithExperiences(std::vector<experience> &instances) = 0;

  virtual bool updateWithExperience(experience &instance) = 0;

  virtual float getStateActionInfo(const std::vector<float> &state, int action, StateActionInfo* retval) = 0;

  virtual MDPModel* getCopy() = 0;
  virtual ~MDPModel() {};
};

class Planner {
public:
  virtual void setModel(MDPModel* model) = 0;

  virtual bool updateModelWithExperience(const std::vector<float>& last,
                                         int act,
                                         const std::vector<float>& curr,
                                         float reward, bool terminal) = 0;

  virtual void planOnNewModel() = 0;

  virtual int getBestAction(const std::vector<float> &s) = 0;

  virtual void savePolicy(const char* filename) {};

  virtual void setSeeding(bool seeding) {};

  virtual void setFirst() {};

  std::vector<float>::iterator
  random_max_element(std::vector<float>::iterator start,
                     std::vector<float>::iterator end) {
    const float Q_EPSILON = 1e-4;
    
    std::vector<float>::iterator max =
    std::max_element(start, end);

    // find # within epsilon of max
    int nfound = 0;
    for (std::vector<float>::iterator it = start; it != end; it++){
      if (fabs(*it - *max) < Q_EPSILON){
        nfound++;
      }
    }
    
    // only 1: take it
    if (nfound == 1)
      return max;

    // take one of close to max at random
    for (std::vector<float>::iterator it = start; it != end; it++){
      if (fabs(*it - *max) < Q_EPSILON){
        if (rng.uniform() < (1.0 / (float)nfound)){
          return it;
        }
        nfound--;
      }
    }
    
    return max;
  };

  virtual ~Planner() {};
  
  Random rng;

};


#endif