Realm

CartPole PPO training

Source-owned programs from the current OA SDK. Rust and Python examples follow their respective public APIs.

Rust · Python

Example

Tu_rl_cart_pole_ppo.rs

1use std::time::Instant;
2
3use oa::ml::{
4 AdamW, CategoricalActorCritic, CategoricalActorCriticConfig, Module,
5 environment::Environment,
6 evaluation::{PolicyEvaluationConfig, PolicyEvaluationMetrics, evaluate_categorical},
7 training::{PpoTrainer, PpoTrainerConfig, RolloutTrainingPhase},
8};
9use sdk::ml::rl::{CartPole, CartPoleConfig};
10
11const ENVIRONMENTS: u32 = 64;
12const HORIZON: usize = 128;
13const ROLLOUTS: u64 = 40;
14const UPDATE_EPOCHS: u64 = 4;
15const TRAINING_SEED: u64 = 0x0a11_ce55;
16const EVALUATION_SEED: u64 = 0x0e7a1;
17const LEARNING_RATE: f32 = 2.5e-4;
18const EVALUATION_HORIZON: usize = 500;
19
20fn evaluate(
21 engine: &oa::Engine,
22 model: &CategoricalActorCritic,
23 seed: u64,
24) -> oa::Result<PolicyEvaluationMetrics> {
25 let mut environment = CartPole::new(
26 engine,
27 CartPoleConfig {
28 environments: ENVIRONMENTS,
29 max_episode_steps: 500,
30 seed,
31 ..CartPoleConfig::default()
32 },
33 )?;
34 let initialized = environment.submit()?;
35 environment.wait(&initialized)?;
36 let result = evaluate_categorical(
37 &mut environment,
38 model,
39 PolicyEvaluationConfig {
40 horizon: EVALUATION_HORIZON,
41 seed,
42 },
43 )?;
44 environment.close()?;
45 Ok(result)
46}
47
48fn collect(environment: &mut CartPole<'_>, trainer: &mut PpoTrainer<'_, '_>) -> oa::Result<()> {
49 if environment.execution().submission_count() == 0
50 && !environment.execution().has_active_recording()
51 {
52 environment.reset(TRAINING_SEED)?;
53 }
54 environment.begin()?;
55 trainer.begin_collection()?;
56 let recorded = (|| {
57 for _ in 0..HORIZON {
58 let policy = trainer.act(environment.observation())?;
59 let transition = environment.step(&policy.action)?;
60 trainer.observe(
61 transition.observation(),
62 transition.next_observation(),
63 transition.reward(),
64 transition.terminated(),
65 transition.truncated(),
66 &policy,
67 )?;
68 environment.reset_completed()?;
69 }
70 trainer.end_collection()
71 })();
72 if let Err(error) = recorded {
73 let _ = environment.cancel();
74 let _ = trainer.abort_collection();
75 return Err(error);
76 }
77 let event = match environment.submit() {
78 Ok(event) => event,
79 Err(error) => {
80 let _ = trainer.abort_collection();
81 return Err(error);
82 }
83 };
84 environment.wait(&event)
85}
86
87fn main() -> oa::Result<()> {
88 let engine = oa::Engine::new()?;
89 oa::Engine::set_default(engine.clone());
90 let model = CategoricalActorCritic::new(CategoricalActorCriticConfig {
91 observation_size: 4,
92 action_count: 2,
93 hidden_size: 64,
94 seed: TRAINING_SEED,
95 })?;
96 let mut optimizer = AdamW::new_with_hyperparameters(
97 model.all_parameters()?,
98 LEARNING_RATE,
99 0.9,
100 0.999,
101 1.0e-8,
102 0.0,
103 )?;
104
105 println!("\nOA reinforcement learning — CartPole PPO");
106 println!(" separate actor/critic: each 4 -> 64 -> 64");
107 println!(" heads: categorical policy 2 · scalar value 1");
108 println!(
109 " rollout: {ENVIRONMENTS} env x {HORIZON} steps · {UPDATE_EPOCHS} PPO epochs · {ROLLOUTS} rollouts"
110 );
111
112 let before = evaluate(&engine, &model, EVALUATION_SEED)?;
113 println!(
114 " before: {:.2} mean completed return ({} episodes)",
115 before.mean_completed_return, before.completed_episodes
116 );
117
118 let mut environment = CartPole::new(
119 &engine,
120 CartPoleConfig {
121 environments: ENVIRONMENTS,
122 max_episode_steps: 500,
123 seed: TRAINING_SEED,
124 ..CartPoleConfig::default()
125 },
126 )?;
127 let started = Instant::now();
128 let snapshot = {
129 let mut trainer = PpoTrainer::new(
130 &engine,
131 &model,
132 &mut optimizer,
133 PpoTrainerConfig {
134 rollouts: ROLLOUTS,
135 horizon: HORIZON,
136 environments: ENVIRONMENTS as usize,
137 update_epochs: UPDATE_EPOCHS,
138 observation_shape: vec![4],
139 seed: TRAINING_SEED,
140 enable_gpu_timing: true,
141 ..PpoTrainerConfig::default()
142 },
143 )?;
144 let mut last_printed = 0;
145 while !trainer.is_done() {
146 if trainer.needs_collection() {
147 collect(&mut environment, &mut trainer)?;
148 }
149 trainer.update()?;
150 let metrics = trainer.metrics();
151 if trainer.phase() != RolloutTrainingPhase::Update
152 && metrics.rollout != last_printed
153 && metrics.rollout % 10 == 0
154 {
155 last_printed = metrics.rollout;
156 println!(
157 " rollout {}/{} · loss {:.5}",
158 metrics.rollout, ROLLOUTS, metrics.total_loss
159 );
160 }
161 }
162 trainer.finish()?
163 };
164 environment.close()?;
165 let wall = started.elapsed();
166
167 let after = evaluate(&engine, &model, EVALUATION_SEED)?;
168 let checkpoint = std::env::temp_dir().join("oars_cart_pole_ppo.oam");
169 oa::ml::save_checkpoint(&checkpoint, &model, &optimizer)?;
170 let restored_model = CategoricalActorCritic::new(CategoricalActorCriticConfig {
171 observation_size: 4,
172 action_count: 2,
173 hidden_size: 64,
174 seed: TRAINING_SEED,
175 })?;
176 let mut restored_optimizer = AdamW::new_with_hyperparameters(
177 restored_model.all_parameters()?,
178 LEARNING_RATE,
179 0.9,
180 0.999,
181 1.0e-8,
182 0.0,
183 )?;
184 oa::ml::load_checkpoint(&checkpoint, &restored_model, &mut restored_optimizer)?;
185 let restored = evaluate(&engine, &restored_model, EVALUATION_SEED)?;
186 std::fs::remove_file(&checkpoint).expect("remove completed CartPole checkpoint");
187
188 println!(
189 " after: {:.2} mean completed return ({} episodes)",
190 after.mean_completed_return, after.completed_episodes
191 );
192 println!(
193 " improvement: {:+.2}",
194 after.mean_completed_return - before.mean_completed_return
195 );
196 println!(
197 " checkpoint: {:.2} restored return · AdamW step {}",
198 restored.mean_completed_return,
199 restored_optimizer.step_count()
200 );
201 println!(
202 " training: {:.2}s wall · {:.2} ms/update wall · {:.2} ms/update GPU · {} transitions",
203 wall.as_secs_f64(),
204 snapshot.wall_ms_per_step(),
205 snapshot.gpu_timing_stats().mean_ms,
206 u64::from(ENVIRONMENTS) * HORIZON as u64 * ROLLOUTS
207 );
208
209 assert!(after.mean_completed_return >= before.mean_completed_return + 25.0);
210 assert!(after.mean_completed_return >= 75.0);
211 assert_eq!(restored.mean_completed_return, after.mean_completed_return);
212 assert_eq!(restored.completed_episodes, after.completed_episodes);
213 assert_eq!(restored_optimizer.step_count(), optimizer.step_count());
214 Ok(())
215}
216

Run Rust tutorials from the OA checkout with cargo run --release --example <name>. Run Python with the environment containing the OA native extension. Consult each program's arguments and capability checks before running it.