Realm

LunarLander PPO

Source-owned programs from the current OA SDK. Rust and Python examples follow their respective public APIs.

Rust · Python

Example

Tu_rl_lunar_lander_ppo.rs

1use std::time::Instant;
2
3use oa::ml::{
4 AdamW, CategoricalActorCritic, CategoricalActorCriticConfig, Module,
5 environment::Environment,
6 training::{PpoTrainer, PpoTrainerConfig, RolloutTrainingPhase},
7};
8use sdk::ml::rl::{
9 LUNAR_OBSERVATION_SIZE, LunarFirstEpisodeEvaluation, LunarFirstEpisodeEvaluationConfig,
10 LunarLander3dConfig, LunarLander3dVector, LunarLander3dVectorConfig, LunarTeacherConfig,
11 collect_ppo_rollout, evaluate_first_episodes, pretrain_scripted_teacher,
12};
13
14const ENVIRONMENTS: u32 = 64;
15const HORIZON: usize = 128;
16const ROLLOUTS: u64 = 40;
17const UPDATE_EPOCHS: u64 = 4;
18const HIDDEN_SIZE: usize = 64;
19const TRAINING_SEED: u64 = 0x1a2b_3c4d;
20const LEARNING_RATE: f32 = 2.5e-4;
21
22fn print_evaluation(label: &str, value: &LunarFirstEpisodeEvaluation) {
23 println!(
24 " {label}: return {:.3} [{:.3}, {:.3}] · safe {}/{} ({:.2}%, Wilson95 {:.2}%)",
25 value.mean_return,
26 value.minimum_return,
27 value.maximum_return,
28 value.safe_landings,
29 value.expected_episodes,
30 value.safe_landing_rate * 100.0,
31 value.wilson_lower_95 * 100.0,
32 );
33 println!(
34 " body {} · hard-foot {} · bounds {} · numerical {} · timeout {} · incomplete {}",
35 value.body_impacts,
36 value.hard_foot_impacts,
37 value.out_of_bounds,
38 value.numerical_failures,
39 value.time_limits,
40 value.incomplete_episodes,
41 );
42}
43
44fn main() -> oa::Result<()> {
45 let engine = oa::Engine::new()?;
46 oa::Engine::set_default(engine.clone());
47 let model = CategoricalActorCritic::new(CategoricalActorCriticConfig {
48 observation_size: LUNAR_OBSERVATION_SIZE,
49 action_count: 8,
50 hidden_size: HIDDEN_SIZE,
51 seed: TRAINING_SEED,
52 })?;
53 let mut optimizer = AdamW::new_with_hyperparameters(
54 model.all_parameters()?,
55 LEARNING_RATE,
56 0.9,
57 0.999,
58 1.0e-8,
59 0.0,
60 )?;
61 let evaluation_config = LunarFirstEpisodeEvaluationConfig::default();
62
63 println!("\nOA reinforcement learning — Lunar Lander 3D flat-v0");
64 println!(
65 " separate actor/critic: each {} -> {} -> {} · categorical actions 8",
66 LUNAR_OBSERVATION_SIZE, HIDDEN_SIZE, HIDDEN_SIZE
67 );
68
69 let before = evaluate_first_episodes(&engine, &model, evaluation_config)?;
70 print_evaluation("untrained", &before);
71
72 if std::env::var_os("OA_LUNAR_RUN_TEACHER_GATE").is_some() {
73 let started = Instant::now();
74 let teacher = pretrain_scripted_teacher(&engine, &model, LunarTeacherConfig::default())?;
75 let after = evaluate_first_episodes(&engine, &model, evaluation_config)?;
76 println!(
77 " teacher: {} samples · {} updates · loss {:.6} -> {:.6} · {:.2}s wall",
78 teacher.samples,
79 teacher.optimizer_steps,
80 teacher.initial_loss,
81 teacher.final_loss,
82 started.elapsed().as_secs_f64(),
83 );
84 print_evaluation("learned", &after);
85 assert!(teacher.final_loss < teacher.initial_loss);
86 assert_eq!(optimizer.step_count(), 0);
87 assert_eq!(after.completed_episodes, after.expected_episodes);
88 assert!(after.safe_landing_rate >= 0.80);
89 assert!(after.wilson_lower_95 >= 0.75);
90 return Ok(());
91 }
92
93 println!(
94 " rollout: {} env x {} steps · {} PPO epochs · {} rollouts",
95 ENVIRONMENTS, HORIZON, UPDATE_EPOCHS, ROLLOUTS
96 );
97 let mut environment = LunarLander3dVector::flat(
98 &engine,
99 LunarLander3dVectorConfig {
100 environments: ENVIRONMENTS,
101 seed: TRAINING_SEED,
102 environment: LunarLander3dConfig::default(),
103 },
104 )?;
105 let started = Instant::now();
106 let snapshot = {
107 let mut trainer = PpoTrainer::new(
108 &engine,
109 &model,
110 &mut optimizer,
111 PpoTrainerConfig {
112 rollouts: ROLLOUTS,
113 horizon: HORIZON,
114 environments: ENVIRONMENTS as usize,
115 update_epochs: UPDATE_EPOCHS,
116 observation_shape: vec![LUNAR_OBSERVATION_SIZE],
117 seed: TRAINING_SEED,
118 enable_gpu_timing: true,
119 ..PpoTrainerConfig::default()
120 },
121 )?;
122 let mut last_printed = 0;
123 while !trainer.is_done() {
124 if trainer.needs_collection() {
125 collect_ppo_rollout(&mut environment, &mut trainer)?;
126 }
127 trainer.update()?;
128 let metrics = trainer.metrics();
129 if trainer.phase() != RolloutTrainingPhase::Update
130 && metrics.rollout != last_printed
131 && (metrics.rollout % 10 == 0 || trainer.is_done())
132 {
133 last_printed = metrics.rollout;
134 println!(
135 " rollout {}/{} · loss {:.6} policy {:.6} value {:.6} entropy {:.6}",
136 metrics.rollout,
137 ROLLOUTS,
138 metrics.total_loss,
139 metrics.policy_loss,
140 metrics.value_loss,
141 metrics.entropy,
142 );
143 }
144 }
145 trainer.finish()?
146 };
147 environment.close()?;
148 let wall = started.elapsed();
149 let after = evaluate_first_episodes(&engine, &model, evaluation_config)?;
150 print_evaluation("trained", &after);
151
152 let checkpoint = std::env::temp_dir().join("oars_lunar_lander_3d_flat_ppo.oam");
153 oa::ml::save_checkpoint(&checkpoint, &model, &optimizer)?;
154 let restored_model = CategoricalActorCritic::new(CategoricalActorCriticConfig {
155 observation_size: LUNAR_OBSERVATION_SIZE,
156 action_count: 8,
157 hidden_size: HIDDEN_SIZE,
158 seed: TRAINING_SEED,
159 })?;
160 let mut restored_optimizer = AdamW::new_with_hyperparameters(
161 restored_model.all_parameters()?,
162 LEARNING_RATE,
163 0.9,
164 0.999,
165 1.0e-8,
166 0.0,
167 )?;
168 oa::ml::load_checkpoint(&checkpoint, &restored_model, &mut restored_optimizer)?;
169 let restored = evaluate_first_episodes(&engine, &restored_model, evaluation_config)?;
170 std::fs::remove_file(&checkpoint).expect("remove completed Lunar checkpoint");
171
172 println!(
173 " training: {:.2}s wall · {:.2} ms/update wall · {:.2} ms/update GPU · {} transitions",
174 wall.as_secs_f64(),
175 snapshot.wall_ms_per_step(),
176 snapshot.gpu_timing_stats().mean_ms,
177 u64::from(ENVIRONMENTS) * HORIZON as u64 * ROLLOUTS,
178 );
179 println!(
180 " checkpoint: action {:016x} · value {:016x} · AdamW step {}",
181 restored.action_trace_digest,
182 restored.value_trace_digest,
183 restored_optimizer.step_count(),
184 );
185 assert_eq!(after.completed_episodes, after.expected_episodes);
186 assert_eq!(restored, after);
187 assert_eq!(restored_optimizer.step_count(), optimizer.step_count());
188 if std::env::var_os("OA_LUNAR_REQUIRE_FLAT_GATE").is_some() {
189 assert!(after.mean_return > before.mean_return);
190 assert!(after.safe_landing_rate >= 0.80);
191 assert!(after.wilson_lower_95 >= 0.75);
192 }
193 Ok(())
194}
195

Run Rust tutorials from the OA checkout with cargo run --release --example <name>. Run Python with the environment containing the OA native extension. Consult each program's arguments and capability checks before running it.