CartPole PPO training
Source-owned programs from the current OA SDK. Rust and Python examples follow their respective public APIs.
Rust · Python
Example
Tu_rl_cart_pole_ppo.rs
1use std::time::Instant;23use oa::ml::{4 AdamW, CategoricalActorCritic, CategoricalActorCriticConfig, Module,5 environment::Environment,6 evaluation::{PolicyEvaluationConfig, PolicyEvaluationMetrics, evaluate_categorical},7 training::{PpoTrainer, PpoTrainerConfig, RolloutTrainingPhase},8};9use sdk::ml::rl::{CartPole, CartPoleConfig};1011const ENVIRONMENTS: u32 = 64;12const HORIZON: usize = 128;13const ROLLOUTS: u64 = 40;14const UPDATE_EPOCHS: u64 = 4;15const TRAINING_SEED: u64 = 0x0a11_ce55;16const EVALUATION_SEED: u64 = 0x0e7a1;17const LEARNING_RATE: f32 = 2.5e-4;18const EVALUATION_HORIZON: usize = 500;1920fn evaluate(21 engine: &oa::Engine,22 model: &CategoricalActorCritic,23 seed: u64,24) -> oa::Result<PolicyEvaluationMetrics> {25 let mut environment = CartPole::new(26 engine,27 CartPoleConfig {28 environments: ENVIRONMENTS,29 max_episode_steps: 500,30 seed,31 ..CartPoleConfig::default()32 },33 )?;34 let initialized = environment.submit()?;35 environment.wait(&initialized)?;36 let result = evaluate_categorical(37 &mut environment,38 model,39 PolicyEvaluationConfig {40 horizon: EVALUATION_HORIZON,41 seed,42 },43 )?;44 environment.close()?;45 Ok(result)46}4748fn collect(environment: &mut CartPole<'_>, trainer: &mut PpoTrainer<'_, '_>) -> oa::Result<()> {49 if environment.execution().submission_count() == 050 && !environment.execution().has_active_recording()51 {52 environment.reset(TRAINING_SEED)?;53 }54 environment.begin()?;55 trainer.begin_collection()?;56 let recorded = (|| {57 for _ in 0..HORIZON {58 let policy = trainer.act(environment.observation())?;59 let transition = environment.step(&policy.action)?;60 trainer.observe(61 transition.observation(),62 transition.next_observation(),63 transition.reward(),64 transition.terminated(),65 transition.truncated(),66 &policy,67 )?;68 environment.reset_completed()?;69 }70 trainer.end_collection()71 })();72 if let Err(error) = recorded {73 let _ = environment.cancel();74 let _ = trainer.abort_collection();75 return Err(error);76 }77 let event = match environment.submit() {78 Ok(event) => event,79 Err(error) => {80 let _ = trainer.abort_collection();81 return Err(error);82 }83 };84 environment.wait(&event)85}8687fn main() -> oa::Result<()> {88 let engine = oa::Engine::new()?;89 oa::Engine::set_default(engine.clone());90 let model = CategoricalActorCritic::new(CategoricalActorCriticConfig {91 observation_size: 4,92 action_count: 2,93 hidden_size: 64,94 seed: TRAINING_SEED,95 })?;96 let mut optimizer = AdamW::new_with_hyperparameters(97 model.all_parameters()?,98 LEARNING_RATE,99 0.9,100 0.999,101 1.0e-8,102 0.0,103 )?;104105 println!("\nOA reinforcement learning — CartPole PPO");106 println!(" separate actor/critic: each 4 -> 64 -> 64");107 println!(" heads: categorical policy 2 · scalar value 1");108 println!(109 " rollout: {ENVIRONMENTS} env x {HORIZON} steps · {UPDATE_EPOCHS} PPO epochs · {ROLLOUTS} rollouts"110 );111112 let before = evaluate(&engine, &model, EVALUATION_SEED)?;113 println!(114 " before: {:.2} mean completed return ({} episodes)",115 before.mean_completed_return, before.completed_episodes116 );117118 let mut environment = CartPole::new(119 &engine,120 CartPoleConfig {121 environments: ENVIRONMENTS,122 max_episode_steps: 500,123 seed: TRAINING_SEED,124 ..CartPoleConfig::default()125 },126 )?;127 let started = Instant::now();128 let snapshot = {129 let mut trainer = PpoTrainer::new(130 &engine,131 &model,132 &mut optimizer,133 PpoTrainerConfig {134 rollouts: ROLLOUTS,135 horizon: HORIZON,136 environments: ENVIRONMENTS as usize,137 update_epochs: UPDATE_EPOCHS,138 observation_shape: vec![4],139 seed: TRAINING_SEED,140 enable_gpu_timing: true,141 ..PpoTrainerConfig::default()142 },143 )?;144 let mut last_printed = 0;145 while !trainer.is_done() {146 if trainer.needs_collection() {147 collect(&mut environment, &mut trainer)?;148 }149 trainer.update()?;150 let metrics = trainer.metrics();151 if trainer.phase() != RolloutTrainingPhase::Update152 && metrics.rollout != last_printed153 && metrics.rollout % 10 == 0154 {155 last_printed = metrics.rollout;156 println!(157 " rollout {}/{} · loss {:.5}",158 metrics.rollout, ROLLOUTS, metrics.total_loss159 );160 }161 }162 trainer.finish()?163 };164 environment.close()?;165 let wall = started.elapsed();166167 let after = evaluate(&engine, &model, EVALUATION_SEED)?;168 let checkpoint = std::env::temp_dir().join("oars_cart_pole_ppo.oam");169 oa::ml::save_checkpoint(&checkpoint, &model, &optimizer)?;170 let restored_model = CategoricalActorCritic::new(CategoricalActorCriticConfig {171 observation_size: 4,172 action_count: 2,173 hidden_size: 64,174 seed: TRAINING_SEED,175 })?;176 let mut restored_optimizer = AdamW::new_with_hyperparameters(177 restored_model.all_parameters()?,178 LEARNING_RATE,179 0.9,180 0.999,181 1.0e-8,182 0.0,183 )?;184 oa::ml::load_checkpoint(&checkpoint, &restored_model, &mut restored_optimizer)?;185 let restored = evaluate(&engine, &restored_model, EVALUATION_SEED)?;186 std::fs::remove_file(&checkpoint).expect("remove completed CartPole checkpoint");187188 println!(189 " after: {:.2} mean completed return ({} episodes)",190 after.mean_completed_return, after.completed_episodes191 );192 println!(193 " improvement: {:+.2}",194 after.mean_completed_return - before.mean_completed_return195 );196 println!(197 " checkpoint: {:.2} restored return · AdamW step {}",198 restored.mean_completed_return,199 restored_optimizer.step_count()200 );201 println!(202 " training: {:.2}s wall · {:.2} ms/update wall · {:.2} ms/update GPU · {} transitions",203 wall.as_secs_f64(),204 snapshot.wall_ms_per_step(),205 snapshot.gpu_timing_stats().mean_ms,206 u64::from(ENVIRONMENTS) * HORIZON as u64 * ROLLOUTS207 );208209 assert!(after.mean_completed_return >= before.mean_completed_return + 25.0);210 assert!(after.mean_completed_return >= 75.0);211 assert_eq!(restored.mean_completed_return, after.mean_completed_return);212 assert_eq!(restored.completed_episodes, after.completed_episodes);213 assert_eq!(restored_optimizer.step_count(), optimizer.step_count());214 Ok(())215}216
Run Rust tutorials from the OA checkout with cargo run --release --example <name>. Run Python with the environment containing the OA native extension. Consult each program's arguments and capability checks before running it.