LunarLander PPO
Source-owned programs from the current OA SDK. Rust and Python examples follow their respective public APIs.
Rust · Python
Example
Tu_rl_lunar_lander_ppo.rs
1use std::time::Instant;23use oa::ml::{4 AdamW, CategoricalActorCritic, CategoricalActorCriticConfig, Module,5 environment::Environment,6 training::{PpoTrainer, PpoTrainerConfig, RolloutTrainingPhase},7};8use sdk::ml::rl::{9 LUNAR_OBSERVATION_SIZE, LunarFirstEpisodeEvaluation, LunarFirstEpisodeEvaluationConfig,10 LunarLander3dConfig, LunarLander3dVector, LunarLander3dVectorConfig, LunarTeacherConfig,11 collect_ppo_rollout, evaluate_first_episodes, pretrain_scripted_teacher,12};1314const ENVIRONMENTS: u32 = 64;15const HORIZON: usize = 128;16const ROLLOUTS: u64 = 40;17const UPDATE_EPOCHS: u64 = 4;18const HIDDEN_SIZE: usize = 64;19const TRAINING_SEED: u64 = 0x1a2b_3c4d;20const LEARNING_RATE: f32 = 2.5e-4;2122fn print_evaluation(label: &str, value: &LunarFirstEpisodeEvaluation) {23 println!(24 " {label}: return {:.3} [{:.3}, {:.3}] · safe {}/{} ({:.2}%, Wilson95 {:.2}%)",25 value.mean_return,26 value.minimum_return,27 value.maximum_return,28 value.safe_landings,29 value.expected_episodes,30 value.safe_landing_rate * 100.0,31 value.wilson_lower_95 * 100.0,32 );33 println!(34 " body {} · hard-foot {} · bounds {} · numerical {} · timeout {} · incomplete {}",35 value.body_impacts,36 value.hard_foot_impacts,37 value.out_of_bounds,38 value.numerical_failures,39 value.time_limits,40 value.incomplete_episodes,41 );42}4344fn main() -> oa::Result<()> {45 let engine = oa::Engine::new()?;46 oa::Engine::set_default(engine.clone());47 let model = CategoricalActorCritic::new(CategoricalActorCriticConfig {48 observation_size: LUNAR_OBSERVATION_SIZE,49 action_count: 8,50 hidden_size: HIDDEN_SIZE,51 seed: TRAINING_SEED,52 })?;53 let mut optimizer = AdamW::new_with_hyperparameters(54 model.all_parameters()?,55 LEARNING_RATE,56 0.9,57 0.999,58 1.0e-8,59 0.0,60 )?;61 let evaluation_config = LunarFirstEpisodeEvaluationConfig::default();6263 println!("\nOA reinforcement learning — Lunar Lander 3D flat-v0");64 println!(65 " separate actor/critic: each {} -> {} -> {} · categorical actions 8",66 LUNAR_OBSERVATION_SIZE, HIDDEN_SIZE, HIDDEN_SIZE67 );6869 let before = evaluate_first_episodes(&engine, &model, evaluation_config)?;70 print_evaluation("untrained", &before);7172 if std::env::var_os("OA_LUNAR_RUN_TEACHER_GATE").is_some() {73 let started = Instant::now();74 let teacher = pretrain_scripted_teacher(&engine, &model, LunarTeacherConfig::default())?;75 let after = evaluate_first_episodes(&engine, &model, evaluation_config)?;76 println!(77 " teacher: {} samples · {} updates · loss {:.6} -> {:.6} · {:.2}s wall",78 teacher.samples,79 teacher.optimizer_steps,80 teacher.initial_loss,81 teacher.final_loss,82 started.elapsed().as_secs_f64(),83 );84 print_evaluation("learned", &after);85 assert!(teacher.final_loss < teacher.initial_loss);86 assert_eq!(optimizer.step_count(), 0);87 assert_eq!(after.completed_episodes, after.expected_episodes);88 assert!(after.safe_landing_rate >= 0.80);89 assert!(after.wilson_lower_95 >= 0.75);90 return Ok(());91 }9293 println!(94 " rollout: {} env x {} steps · {} PPO epochs · {} rollouts",95 ENVIRONMENTS, HORIZON, UPDATE_EPOCHS, ROLLOUTS96 );97 let mut environment = LunarLander3dVector::flat(98 &engine,99 LunarLander3dVectorConfig {100 environments: ENVIRONMENTS,101 seed: TRAINING_SEED,102 environment: LunarLander3dConfig::default(),103 },104 )?;105 let started = Instant::now();106 let snapshot = {107 let mut trainer = PpoTrainer::new(108 &engine,109 &model,110 &mut optimizer,111 PpoTrainerConfig {112 rollouts: ROLLOUTS,113 horizon: HORIZON,114 environments: ENVIRONMENTS as usize,115 update_epochs: UPDATE_EPOCHS,116 observation_shape: vec![LUNAR_OBSERVATION_SIZE],117 seed: TRAINING_SEED,118 enable_gpu_timing: true,119 ..PpoTrainerConfig::default()120 },121 )?;122 let mut last_printed = 0;123 while !trainer.is_done() {124 if trainer.needs_collection() {125 collect_ppo_rollout(&mut environment, &mut trainer)?;126 }127 trainer.update()?;128 let metrics = trainer.metrics();129 if trainer.phase() != RolloutTrainingPhase::Update130 && metrics.rollout != last_printed131 && (metrics.rollout % 10 == 0 || trainer.is_done())132 {133 last_printed = metrics.rollout;134 println!(135 " rollout {}/{} · loss {:.6} policy {:.6} value {:.6} entropy {:.6}",136 metrics.rollout,137 ROLLOUTS,138 metrics.total_loss,139 metrics.policy_loss,140 metrics.value_loss,141 metrics.entropy,142 );143 }144 }145 trainer.finish()?146 };147 environment.close()?;148 let wall = started.elapsed();149 let after = evaluate_first_episodes(&engine, &model, evaluation_config)?;150 print_evaluation("trained", &after);151152 let checkpoint = std::env::temp_dir().join("oars_lunar_lander_3d_flat_ppo.oam");153 oa::ml::save_checkpoint(&checkpoint, &model, &optimizer)?;154 let restored_model = CategoricalActorCritic::new(CategoricalActorCriticConfig {155 observation_size: LUNAR_OBSERVATION_SIZE,156 action_count: 8,157 hidden_size: HIDDEN_SIZE,158 seed: TRAINING_SEED,159 })?;160 let mut restored_optimizer = AdamW::new_with_hyperparameters(161 restored_model.all_parameters()?,162 LEARNING_RATE,163 0.9,164 0.999,165 1.0e-8,166 0.0,167 )?;168 oa::ml::load_checkpoint(&checkpoint, &restored_model, &mut restored_optimizer)?;169 let restored = evaluate_first_episodes(&engine, &restored_model, evaluation_config)?;170 std::fs::remove_file(&checkpoint).expect("remove completed Lunar checkpoint");171172 println!(173 " training: {:.2}s wall · {:.2} ms/update wall · {:.2} ms/update GPU · {} transitions",174 wall.as_secs_f64(),175 snapshot.wall_ms_per_step(),176 snapshot.gpu_timing_stats().mean_ms,177 u64::from(ENVIRONMENTS) * HORIZON as u64 * ROLLOUTS,178 );179 println!(180 " checkpoint: action {:016x} · value {:016x} · AdamW step {}",181 restored.action_trace_digest,182 restored.value_trace_digest,183 restored_optimizer.step_count(),184 );185 assert_eq!(after.completed_episodes, after.expected_episodes);186 assert_eq!(restored, after);187 assert_eq!(restored_optimizer.step_count(), optimizer.step_count());188 if std::env::var_os("OA_LUNAR_REQUIRE_FLAT_GATE").is_some() {189 assert!(after.mean_return > before.mean_return);190 assert!(after.safe_landing_rate >= 0.80);191 assert!(after.wilson_lower_95 >= 0.75);192 }193 Ok(())194}195
Run Rust tutorials from the OA checkout with cargo run --release --example <name>. Run Python with the environment containing the OA native extension. Consult each program's arguments and capability checks before running it.