PO_ParallelETUCT.hh

Go to the documentation of this file.

#ifndef _PO_ParallelETUCT_HH_
#define _PO_ParallelETUCT_HH_

#include <rl_common/Random.h>
#include <rl_common/core.hh>
#include <rl_common/ExperienceFile.hh>

#include "../Models/FactoredModel.hh"
#include "../Models/C45Tree.hh"

#include <set>
#include <vector>
#include <map>
#include <sstream>
#include <deque>

void* poParallelSearchStart(void* arg);

void* poParallelModelLearningStart(void* arg);

class PO_ParallelETUCT: public Planner {
public:

  PO_ParallelETUCT(int numactions, float gamma, float rrange, float lambda,
                   int MAX_ITER, float MAX_TIME, int MAX_DEPTH,  int modelType,
                   const std::vector<float> &featmax, const std::vector<float> &featmin,
                   const std::vector<int> &statesPerDim, bool trackActual, int historySize, 
                   Random rng = Random());
  
  PO_ParallelETUCT(const PO_ParallelETUCT &);

  virtual ~PO_ParallelETUCT();

  virtual void setModel(MDPModel* model);
  virtual bool updateModelWithExperience(const std::vector<float> &last, 
                                         int act, 
                                         const std::vector<float> &curr, 
                                         float reward, bool term);
  virtual void planOnNewModel();
  virtual int getBestAction(const std::vector<float> &s);

  virtual void setSeeding(bool seed);
  virtual void setFirst();

  bool PLANNERDEBUG;
  bool POLICYDEBUG; //= false; //true;
  bool MODELDEBUG;
  bool ACTDEBUG;
  bool UCTDEBUG;
  bool PTHREADDEBUG;
  bool ATHREADDEBUG;
  bool MTHREADDEBUG;
  bool TIMINGDEBUG;
  bool REALSTATEDEBUG;
  bool HISTORYDEBUG;

  MDPModel* model;
  
  MDPModel* modelcopy;

  typedef const std::vector<float> *state_t;



  // parallel stuff

  // tell if thread started
  bool modelThreadStarted;
  bool planThreadStarted;

  // the threads
  pthread_t planThread;
  
  pthread_t modelThread;

  // some variables that are locked
  std::vector<experience> expList;
  
  state_t discPlanState;
  
  std::vector<float> actualPlanState;
  
  state_t startState;

  // lock over simple objects
  pthread_mutex_t update_mutex;
  pthread_mutex_t nactions_mutex;
  pthread_mutex_t plan_state_mutex;
  pthread_mutex_t model_mutex;
  pthread_mutex_t list_mutex;
  pthread_mutex_t history_mutex;

  pthread_mutex_t statespace_mutex;

  // condition for when list is updated
  pthread_cond_t list_cond; 


  float uctSearch(const std::vector<float> &actS, state_t state, int depth);
  
  std::vector<float> selectRandomState();
  
  void parallelModelLearning();
  
  void parallelSearch();
  
  void loadPolicy(const char* filename);
  
  void logValues(ofstream *of, int xmin, int xmax, int ymin, int ymax);
  
  std::vector<float> addVec(const std::vector<float> &a, const std::vector<float> &b);
  
  std::vector<float> subVec(const std::vector<float> &a, const std::vector<float> &b);

protected:


  struct state_info;
  struct model_info;

  struct state_samples {
    std::vector<state_t> samples;
  };

  struct state_info {

    // data filled in from models
    StateActionInfo* model;

    // q values from policy creation
    std::vector<float> Q;

    // uct experience data
    int uctVisits;
    std::vector<int> uctActions;
    short unsigned int visited;
    short unsigned int id;

    // needs update
    bool needsUpdate;

    // mutex for model info, samples, everything else
    pthread_mutex_t statemodel_mutex;
    pthread_mutex_t stateinfo_mutex;

  };

  void initStateInfo(state_t s,state_info* info, int id);
  
  state_t canonicalize(const std::vector<float> &s);

  void deleteInfo(state_info* info);
  
  void createPolicy();
  
  void printStates();
  
  void calculateReachableStates();
  
  void removeUnreachableStates();

  void updateStateActionFromModel(state_t s, int a, state_info* info);
  
  void updateStateActionHistoryFromModel(const std::vector<float> modState, int a, StateActionInfo *newModel);

  double getSeconds();

  // uct stuff
  void resetAndUpdateStateActions();

  std::vector<float> simulateNextState(const std::vector<float> &actS, state_t state, state_info* info, int action, float* reward, bool* term);
  
  int selectUCTAction(state_info* info);
  
  void canonNextStates(StateActionInfo* modelInfo);
  
  void initStates();
  
  void fillInState(std::vector<float>s, int depth);

  virtual void savePolicy(const char* filename);
  
  std::vector<float> discretizeState(const std::vector<float> &s);

private:

  std::set<std::vector<float> > statespace;

  std::map<state_t, state_info> statedata;

  std::vector<float> featmax;
  std::vector<float> featmin;

  std::deque<float> saHistory;

  state_t prevstate;
  int prevact;
  state_info* previnfo;

  double planTime;
  double initTime;
  double setTime;
  bool seedMode;

  int nstates;
  int nsaved;
  int nactions;
  int lastUpdate;

  bool timingType;

  const int numactions;
  const float gamma;
  const float rrange;
  const float lambda;

  const int MAX_ITER;
  const float MAX_TIME;
  const int MAX_DEPTH;
  const int modelType;
  const std::vector<int> &statesPerDim;
  const bool trackActual;
  const int HISTORY_SIZE;
  const int HISTORY_FL_SIZE;

  ExperienceFile expfile;
};

#endif