Classes
class	PolicyNetwork

class	QValueNetwork
	— Q-value and policy networks More...

class	ReplayItem

Functions
def	rendertrial (maxiter=NSTEPS, verbose=True)

Variables
	batch

int	BATCH_SIZE = 64

	d_batch = np.vstack([b.done for b in batch])

float	DECAY_RATE = 0.99

bool	done = False

	env = Pendulum(1)
	— Environment More...

	feed_dict

list	h_qva = []

list	h_rwd = []
	History of search. More...

list	h_ste = []

tuple	maxq

	n_init = tflearn.initializations.truncated_normal(seed=RANDOM_SEED)

int	NEPISODES = 100
	— Hyper paramaters More...

int	NH1 = 250

int	NSTEPS = 100

	NU = env.nu

	NX = env.nobs

	optim

	policy = PolicyNetwork().setupOptim()
	— Tensor flow initialization More...

float	POLICY_LEARNING_RATE = 0.0001

	policyTarget = PolicyNetwork().setupTargetAssign(policy)

	q2_batch

	qgrad

	qref_batch = r_batch + (not d_batch) * (DECAY_RATE * q2_batch)

	qvalue = QValueNetwork().setupOptim()

float	QVALUE_LEARNING_RATE = 0.001

	qvalueTarget = QValueNetwork().setupTargetAssign(qvalue)

	r

	r_batch = np.vstack([b.reward for b in batch])

	RANDOM_SEED = int((time.time() % 10) * 1000)
	— Random seed More...

int	REPLAY_SIZE = 10000

	replayDeque = deque()

float	rsum = 0.0

	sess = tf.InteractiveSession()

	u = sess.run(policy.policy, feed_dict={policy.x: x})

	u2_batch

	u_batch = np.vstack([b.u for b in batch])

	u_init = tflearn.initializations.uniform(minval=-0.003, maxval=0.003, seed=RANDOM_SEED)

	u_targ = sess.run(policy.policy, feed_dict={policy.x: x_batch})

float	UPDATE_RATE = 0.01

	withSinCos

	x = env.reset().T
	— Training More...

	x2 = x2.T

	x2_batch = np.vstack([b.x2 for b in batch])

	x_batch = np.vstack([b.x for b in batch])

Detailed Description

Deep actor-critic network,
From "Continuous control with deep reinforcement learning",
by Lillicrap et al, arXiv:1509.02971

Function Documentation

◆ rendertrial()

def continuous.rendertrial	(	maxiter = `NSTEPS`,
		verbose = `True`
	)

Definition at line 153 of file continuous.py.

Variable Documentation

◆ batch

continuous.batch

Initial value:

 =  random.sample(
                 replayDeque, BATCH_SIZE
             )

Definition at line 198 of file continuous.py.

◆ BATCH_SIZE

int continuous.BATCH_SIZE = 64

Definition at line 35 of file continuous.py.

◆ d_batch

continuous.d_batch = np.vstack([b.done for b in batch])

Definition at line 204 of file continuous.py.

◆ DECAY_RATE

float continuous.DECAY_RATE = 0.99

Definition at line 32 of file continuous.py.

◆ done

bool continuous.done = False

Definition at line 185 of file continuous.py.

◆ env

continuous.env = Pendulum(1)

— Environment

Definition at line 39 of file continuous.py.

◆ feed_dict

continuous.feed_dict

Definition at line 220 of file continuous.py.

◆ h_qva

list continuous.h_qva = []

Definition at line 172 of file continuous.py.

◆ h_rwd

list continuous.h_rwd = []

History of search.

Definition at line 171 of file continuous.py.

◆ h_ste

list continuous.h_ste = []

Definition at line 173 of file continuous.py.

◆ maxq

tuple continuous.maxq

Initial value:

 =  (
         np.max(
             sess.run(qvalue.qvalue, feed_dict={qvalue.x: x_batch, qvalue.u: u_batch})
         )
         if "x_batch" in locals()
         else 0
     )

Definition at line 244 of file continuous.py.

◆ n_init

continuous.n_init = tflearn.initializations.truncated_normal(seed=RANDOM_SEED)

Definition at line 24 of file continuous.py.

◆ NEPISODES

int continuous.NEPISODES = 100

— Hyper paramaters

Definition at line 28 of file continuous.py.

◆ NH1

int continuous.NH1 = 250

Definition at line 36 of file continuous.py.

◆ NSTEPS

int continuous.NSTEPS = 100

Definition at line 29 of file continuous.py.

◆ NU

continuous.NU = env.nu

Definition at line 42 of file continuous.py.

◆ NX

continuous.NX = env.nobs

Definition at line 41 of file continuous.py.

◆ optim

continuous.optim

Definition at line 234 of file continuous.py.

◆ policy

continuous.policy = PolicyNetwork().setupOptim()

— Tensor flow initialization

Definition at line 139 of file continuous.py.

◆ POLICY_LEARNING_RATE

float continuous.POLICY_LEARNING_RATE = 0.0001

Definition at line 31 of file continuous.py.

◆ policyTarget

continuous.policyTarget = PolicyNetwork().setupTargetAssign(policy)

Definition at line 140 of file continuous.py.

◆ q2_batch

continuous.q2_batch

Initial value:

 =  sess.run(
                 qvalueTarget.qvalue,
                 feed_dict={qvalueTarget.x: x2_batch, qvalueTarget.u: u2_batch},
             )

Definition at line 211 of file continuous.py.

◆ qgrad

continuous.qgrad

Initial value:

 =  sess.run(
                 qvalue.gradient, feed_dict={qvalue.x: x_batch, qvalue.u: u_targ}
             )

Definition at line 229 of file continuous.py.

◆ qref_batch

continuous.qref_batch = r_batch + (not d_batch) * (DECAY_RATE * q2_batch)

Definition at line 215 of file continuous.py.

◆ qvalue

continuous.qvalue = QValueNetwork().setupOptim()

Definition at line 142 of file continuous.py.

◆ QVALUE_LEARNING_RATE

float continuous.QVALUE_LEARNING_RATE = 0.001

Definition at line 30 of file continuous.py.

◆ qvalueTarget

continuous.qvalueTarget = QValueNetwork().setupTargetAssign(qvalue)

Definition at line 143 of file continuous.py.

◆ r

continuous.r

Definition at line 183 of file continuous.py.

◆ r_batch

continuous.r_batch = np.vstack([b.reward for b in batch])

Definition at line 203 of file continuous.py.

◆ RANDOM_SEED

continuous.RANDOM_SEED = int((time.time() % 10) * 1000)

— Random seed

Definition at line 19 of file continuous.py.

◆ REPLAY_SIZE

int continuous.REPLAY_SIZE = 10000

Definition at line 34 of file continuous.py.

◆ replayDeque

continuous.replayDeque = deque()

Definition at line 135 of file continuous.py.

◆ rsum

float continuous.rsum = 0.0

Definition at line 178 of file continuous.py.

◆ sess

continuous.sess = tf.InteractiveSession()

Definition at line 145 of file continuous.py.

◆ u

continuous.u = sess.run(policy.policy, feed_dict={policy.x: x})

Definition at line 181 of file continuous.py.

◆ u2_batch

continuous.u2_batch

Initial value:

 =  sess.run(
                 policyTarget.policy, feed_dict={policyTarget.x: x2_batch}
             )

Definition at line 208 of file continuous.py.

◆ u_batch

continuous.u_batch = np.vstack([b.u for b in batch])

Definition at line 202 of file continuous.py.

◆ u_init

continuous.u_init = tflearn.initializations.uniform(minval=-0.003, maxval=0.003, seed=RANDOM_SEED)

Definition at line 25 of file continuous.py.

◆ u_targ

continuous.u_targ = sess.run(policy.policy, feed_dict={policy.x: x_batch})

Definition at line 228 of file continuous.py.

◆ UPDATE_RATE

float continuous.UPDATE_RATE = 0.01

Definition at line 33 of file continuous.py.

◆ withSinCos

continuous.withSinCos

Definition at line 40 of file continuous.py.

◆ x

continuous.x = env.reset().T

— Training

Definition at line 177 of file continuous.py.

◆ x2

continuous.x2 = x2.T

Definition at line 183 of file continuous.py.

◆ x2_batch

continuous.x2_batch = np.vstack([b.x2 for b in batch])

Definition at line 205 of file continuous.py.

◆ x_batch

continuous.x_batch = np.vstack([b.x for b in batch])

Definition at line 201 of file continuous.py.

Classes

Functions

Variables

Detailed Description

Function Documentation

◆ rendertrial()

Variable Documentation

◆ batch

◆ BATCH_SIZE

◆ d_batch

◆ DECAY_RATE

◆ done

◆ env

◆ feed_dict

◆ h_qva

◆ h_rwd

◆ h_ste

◆ maxq

◆ n_init

◆ NEPISODES

◆ NH1

◆ NSTEPS

◆ NU

◆ NX

◆ optim

◆ policy

◆ POLICY_LEARNING_RATE

◆ policyTarget

◆ q2_batch

◆ qgrad

◆ qref_batch

◆ qvalue

◆ QVALUE_LEARNING_RATE

◆ qvalueTarget

◆ r

◆ r_batch

◆ RANDOM_SEED

◆ REPLAY_SIZE

◆ replayDeque

◆ rsum

◆ sess

◆ u

◆ u2_batch

◆ u_batch

◆ u_init

◆ u_targ

◆ UPDATE_RATE

◆ withSinCos

◆ x

◆ x2

◆ x2_batch

◆ x_batch