ModelBasedAgent.hh

Go to the documentation of this file.

#ifndef _MODELBASED_HH_
#define _MODELBASED_HH_

#include <rl_common/Random.h>
#include <rl_common/core.hh>

#include <set>
#include <vector>
#include <map>

class ModelBasedAgent: public Agent {
public:
  ModelBasedAgent(int numactions, float gamma, float rmax, float rrange, 
                  int modelType, int exploreType, 
                  int predType, int nModels, int plannerType,
                  float epsilon, float lambda, float MAX_TIME,
                  float m, const std::vector<float> &featmin, 
                  const std::vector<float> &featmax,
                  int statesPerDim, int history, float v, float n,
                  bool depTrans, bool relTrans, float featPct,
                  bool stoch, bool episodic, Random rng = Random());

  ModelBasedAgent(int numactions, float gamma, float rmax, float rrange, 
                  int modelType, int exploreType, 
                  int predType, int nModels, int plannerType,
                  float epsilon, float lambda, float MAX_TIME,
                  float m, const std::vector<float> &featmin, 
                  const std::vector<float> &featmax,
                  std::vector<int> statesPerDim, int history, float v, float n,
                  bool depTrans, bool relTrans, float featPct,
                  bool stoch, bool episodic, Random rng = Random());
  
  void initParams();

  ModelBasedAgent(const ModelBasedAgent &);

  virtual ~ModelBasedAgent();

  virtual int first_action(const std::vector<float> &s);
  virtual int next_action(float r, const std::vector<float> &s);
  virtual void last_action(float r);
  virtual void seedExp(std::vector<experience> seeds);
  virtual void setDebug(bool d);
  virtual void savePolicy(const char* filename);

  void logValues(ofstream *of, int xmin, int xmax, int ymin, int ymax);

  bool AGENTDEBUG;
  bool POLICYDEBUG; //= false; //true;
  bool ACTDEBUG;
  bool SIMPLEDEBUG;
  bool TIMEDEBUG;

  bool seeding;

  MDPModel* model;

  Planner* planner;

  float planningTime;
  float modelUpdateTime;
  float actionTime;

  std::vector<float> featmin;
  std::vector<float> featmax;

protected:

  typedef const std::vector<float> *state_t;

  void saveStateAndAction(const std::vector<float> &s, int act);

  int chooseAction(const std::vector<float> &s);

  void initModel(int nfactors);

  void initPlanner();

  void updateWithNewExperience(const std::vector<float> &last, 
                               const std::vector<float> & curr, 
                               int lastact, float reward, bool term);

  double getSeconds();

private:

  std::vector<float> prevstate;
  int prevact;

  int nstates;
  int nactions; 
 
  bool modelNeedsUpdate;
  int lastUpdate;

  int BATCH_FREQ;

  bool modelChanged;

  const int numactions;
  const float gamma;

  const float rmax;
  const float rrange;
  const float qmax;
  const int modelType;
  const int exploreType;
  const int predType;
  const int nModels;
  const int plannerType;

  const float epsilon;
  const float lambda;
  const float MAX_TIME;

  const float M;
  const std::vector<int> statesPerDim;
  const int history;
  const float v;
  const float n;
  const bool depTrans;
  const bool relTrans;
  const float featPct;
  const bool stoch;
  const bool episodic;

  Random rng;

};

#endif