#include <ModelBasedAgent.hh>

Inheritance diagram for ModelBasedAgent:

Public Member Functions
virtual int	first_action (const std::vector< float > &s)
void	initParams ()
virtual void	last_action (float r)
void	logValues (ofstream *of, int xmin, int xmax, int ymin, int ymax)
	ModelBasedAgent (int numactions, float gamma, float rmax, float rrange, int modelType, int exploreType, int predType, int nModels, int plannerType, float epsilon, float lambda, float MAX_TIME, float m, const std::vector< float > &featmin, const std::vector< float > &featmax, int statesPerDim, int history, float v, float n, bool depTrans, bool relTrans, float featPct, bool stoch, bool episodic, Random rng=Random())
	ModelBasedAgent (int numactions, float gamma, float rmax, float rrange, int modelType, int exploreType, int predType, int nModels, int plannerType, float epsilon, float lambda, float MAX_TIME, float m, const std::vector< float > &featmin, const std::vector< float > &featmax, std::vector< int > statesPerDim, int history, float v, float n, bool depTrans, bool relTrans, float featPct, bool stoch, bool episodic, Random rng=Random())
	ModelBasedAgent (const ModelBasedAgent &)
virtual int	next_action (float r, const std::vector< float > &s)
virtual void	savePolicy (const char *filename)
virtual void	seedExp (std::vector< experience > seeds)
virtual void	setDebug (bool d)
virtual	~ModelBasedAgent ()
Public Attributes
bool	ACTDEBUG
float	actionTime
bool	AGENTDEBUG
std::vector< float >	featmax
std::vector< float >	featmin
MDPModel *	model
float	modelUpdateTime
Planner *	planner
float	planningTime
bool	POLICYDEBUG
bool	seeding
bool	SIMPLEDEBUG
bool	TIMEDEBUG
Protected Types
typedef const std::vector < float > *	state_t
Protected Member Functions
int	chooseAction (const std::vector< float > &s)
double	getSeconds ()
void	initModel (int nfactors)
void	initPlanner ()
void	saveStateAndAction (const std::vector< float > &s, int act)
void	updateWithNewExperience (const std::vector< float > &last, const std::vector< float > &curr, int lastact, float reward, bool term)
Private Attributes
int	BATCH_FREQ
const bool	depTrans
const bool	episodic
const float	epsilon
const int	exploreType
const float	featPct
const float	gamma
const int	history
const float	lambda
int	lastUpdate
const float	M
const float	MAX_TIME
bool	modelChanged
bool	modelNeedsUpdate
const int	modelType
const float	n
int	nactions
const int	nModels
int	nstates
const int	numactions
const int	plannerType
const int	predType
int	prevact
std::vector< float >	prevstate
const float	qmax
const bool	relTrans
const float	rmax
Random	rng
const float	rrange
const std::vector< int >	statesPerDim
const bool	stoch
const float	v

Detailed Description

Code for a model based agent, that can use any model and planner that meet the interface requirements

Definition at line 17 of file ModelBasedAgent.hh.

Member Typedef Documentation

typedef const std::vector<float>* ModelBasedAgent::state_t [protected]

The implementation maps all sensations to a set of canonical pointers, which serve as the internal representation of environment state.

Definition at line 135 of file ModelBasedAgent.hh.

Constructor & Destructor Documentation

ModelBasedAgent::ModelBasedAgent	(	int	numactions,
		float	gamma,
		float	rmax,
		float	rrange,
		int	modelType,
		int	exploreType,
		int	predType,
		int	nModels,
		int	plannerType,
		float	epsilon,
		float	lambda,
		float	MAX_TIME,
		float	m,
		const std::vector< float > &	featmin,
		const std::vector< float > &	featmax,
		int	statesPerDim,
		int	history,
		float	v,
		float	n,
		bool	depTrans,
		bool	relTrans,
		float	featPct,
		bool	stoch,
		bool	episodic,
		Random	rng = `Random()`
	)

Standard constructor

Parameters:

numactions	The number of possible actions
gamma	The discount factor
rmax	max reward value, given out for unknown states
rrange	range between max and min reward in domain
modelType	specifies model type
exploreType	specifies exploration type
predType	specifies how to combine multiple models
nModels	number of models in ensemble
plannerType	specifies planner type
epsilon	used for epsilon-greedy action selection
lambda	used for eligibility traces in uct planning
MAX_TIME	amount of time the uct planners are given for planning
m	# visits required for a state to become known when doing rmax exploration
featmin	min values of each feature
featmax	max values of each feature
statesPerDim	# of values to discretize each feature into
history	# of previous actions to use for delayed domains
b\v	bonus reward used when models disagree
n	bonus reward used for novel states
depTrans	assume dependent or indep. feature transitions
relTrans	model transitions relatively vs absolutely
featPct	pct of feature to remove from set used for each split in tree
stoch	is the domain stochastic?
episodic	is the domain episodic?
rng	Initial state of the random number generator to use

Definition at line 57 of file ModelBasedAgent.cc.

ModelBasedAgent::ModelBasedAgent	(	int	numactions,
		float	gamma,
		float	rmax,
		float	rrange,
		int	modelType,
		int	exploreType,
		int	predType,
		int	nModels,
		int	plannerType,
		float	epsilon,
		float	lambda,
		float	MAX_TIME,
		float	m,
		const std::vector< float > &	featmin,
		const std::vector< float > &	featmax,
		std::vector< int >	statesPerDim,
		int	history,
		float	v,
		float	n,
		bool	depTrans,
		bool	relTrans,
		float	featPct,
		bool	stoch,
		bool	episodic,
		Random	rng = `Random()`
	)

Standard constructor

Parameters:

numactions	The number of possible actions
gamma	The discount factor
rmax	max reward value, given out for unknown states
rrange	range between max and min reward in domain
modelType	specifies model type
exploreType	specifies exploration type
predType	specifies how to combine multiple models
nModels	number of models in ensemble
plannerType	specifies planner type
epsilon	used for epsilon-greedy action selection
lambda	used for eligibility traces in uct planning
MAX_TIME	amount of time the uct planners are given for planning
m	# visits required for a state to become known when doing rmax exploration
featmin	min values of each feature
featmax	max values of each feature
statesPerDim	# of values to discretize each feature into
history	# of previous actions to use for delayed domains
b	bonus reward used when models disagree
depTrans	assume dependent or indep. feature transitions
relTrans	model transitions relatively vs absolutely
featPct	pct of feature to remove from set used for each split in tree
stoch	is the domain stochastic?
episodic	is the domain episodic?
rng	Initial state of the random number generator to use

Definition at line 26 of file ModelBasedAgent.cc.

ModelBasedAgent::ModelBasedAgent ( const ModelBasedAgent & )

Unimplemented copy constructor: internal state cannot be simply copied.

ModelBasedAgent::~ModelBasedAgent ( ) [virtual]

Definition at line 130 of file ModelBasedAgent.cc.

Member Function Documentation

int ModelBasedAgent::chooseAction ( const std::vector< float > & s ) [protected]

Select action from the given state

Definition at line 386 of file ModelBasedAgent.cc.

int ModelBasedAgent::first_action ( const std::vector< float > & s ) [virtual]

Implements Agent.

Definition at line 138 of file ModelBasedAgent.cc.

double ModelBasedAgent::getSeconds ( ) [protected]

Get the current time in seconds

Definition at line 434 of file ModelBasedAgent.cc.

void ModelBasedAgent::initModel ( int nfactors ) [protected]

Initialize the model with the given # of features

Definition at line 217 of file ModelBasedAgent.cc.

void ModelBasedAgent::initParams ( )

Init params for both constructors

Definition at line 87 of file ModelBasedAgent.cc.

void ModelBasedAgent::initPlanner ( ) [protected]

Initialize the planner

Definition at line 284 of file ModelBasedAgent.cc.

void ModelBasedAgent::last_action ( float r ) [virtual]

Implements Agent.

Definition at line 198 of file ModelBasedAgent.cc.

void ModelBasedAgent::logValues	(	ofstream *	of,
		int	xmin,
		int	xmax,
		int	ymin,
		int	ymax
	)

Output value function to a file

Definition at line 491 of file ModelBasedAgent.cc.

int ModelBasedAgent::next_action	(	float	r,
		const std::vector< float > &	s
	)		`[virtual]`

Implements Agent.

Definition at line 166 of file ModelBasedAgent.cc.

void ModelBasedAgent::savePolicy ( const char * filename ) [virtual]

Reimplemented from Agent.

Definition at line 485 of file ModelBasedAgent.cc.

void ModelBasedAgent::saveStateAndAction	(	const std::vector< float > &	s,
		int	act
	)		`[protected]`

Saves state and action to use for update on next action

Definition at line 421 of file ModelBasedAgent.cc.

void ModelBasedAgent::seedExp ( std::vector< experience > seeds ) [virtual]

Reimplemented from Agent.

Definition at line 442 of file ModelBasedAgent.cc.

void ModelBasedAgent::setDebug ( bool d ) [virtual]

Implements Agent.

Definition at line 481 of file ModelBasedAgent.cc.

void ModelBasedAgent::updateWithNewExperience	(	const std::vector< float > &	last,
		const std::vector< float > &	curr,
		int	lastact,
		float	reward,
		bool	term
	)		`[protected]`