Sarsa.hh

Go to the documentation of this file.

#ifndef _SARSA_HH_
#define _SARSA_HH_

#include <rl_common/Random.h>
#include <rl_common/core.hh>

#include <map>
#include <set>
#include <vector>

class Sarsa: public Agent {
public:
  Sarsa(int numactions, float gamma,
        float initialvalue, float alpha, float epsilon, float lambda, 
        Random rng = Random());

  Sarsa(const Sarsa &);

  virtual ~Sarsa();

  virtual int first_action(const std::vector<float> &s);
  virtual int next_action(float r, const std::vector<float> &s);
  virtual void last_action(float r);
  virtual void setDebug(bool d);
  virtual void seedExp(std::vector<experience>);
  virtual void savePolicy(const char* filename);

  void printState(const std::vector<float> &s);
  float getValue(std::vector<float> state);
  
  std::vector<float>::iterator random_max_element(
                                                   std::vector<float>::iterator start,
                                                   std::vector<float>::iterator end);

  void logValues(ofstream *of, int xmin, int xmax, int ymin, int ymax);

protected:
  typedef const std::vector<float> *state_t;

  state_t canonicalize(const std::vector<float> &s);

private:
  std::set<std::vector<float> > statespace;

  std::map<state_t, std::vector<float> > Q;
  std::map<state_t, std::vector<float> > eligibility;

  const int numactions;
  const float gamma;

  const float initialvalue;
  const float alpha;
  const float epsilon;
  const float lambda;

  Random rng;
  float *currentq;

  bool ACTDEBUG;
  bool ELIGDEBUG;
};

#endif