Realm

CartPole rollout

Source-owned programs from the current OA SDK. Rust and Python examples follow their respective public APIs.

Rust · Python

Example

Tu_rl_cart_pole_rollout.rs

1use oa::ml::{
2 RolloutBuffer, RolloutConfig, RolloutTransition, advantage, environment::Environment, policy,
3};
4use sdk::ml::rl::{CartPole, CartPoleConfig};
5
6const ENVIRONMENTS: usize = 32;
7const HORIZON: usize = 64;
8const SEED: u64 = 20_260_716;
9
10fn main() -> oa::Result<()> {
11 let engine = oa::Engine::new()?;
12 oa::Engine::set_default(engine.clone());
13 let mut environment = CartPole::new(
14 &engine,
15 CartPoleConfig {
16 environments: ENVIRONMENTS as u32,
17 max_episode_steps: 500,
18 seed: SEED,
19 ..CartPoleConfig::default()
20 },
21 )?;
22 let mut rollout = RolloutBuffer::new(
23 &engine,
24 RolloutConfig {
25 time: HORIZON,
26 environments: ENVIRONMENTS,
27 observation_shape: vec![4],
28 },
29 )?;
30 let policy_weight =
31 oa::Matrix::new_from_f32([2, 4], &[0.0, -0.1, -4.0, -1.0, 0.0, 0.1, 4.0, 1.0])?;
32 let value = oa::Matrix::new_from_f32([ENVIRONMENTS], &[0.0; ENVIRONMENTS])?;
33
34 println!("\nOA reinforcement learning — vectorized GPU CartPole rollout");
35 println!(
36 " environments: {ENVIRONMENTS} · horizon: {HORIZON} · transitions: {}",
37 ENVIRONMENTS * HORIZON
38 );
39
40 rollout.reset()?;
41 for step in 0..HORIZON {
42 let logits = oa::matrix::mat_mul_nt(environment.observation(), &policy_weight)?;
43 let selected = policy::sample_categorical(&logits, &value, SEED + step as u64 + 1)?;
44 let transition = environment.step(&selected.action)?;
45 rollout.append(&RolloutTransition::new(
46 transition.observation().clone(),
47 selected.action,
48 transition.reward().clone(),
49 selected.value,
50 value.clone(),
51 selected.log_probability,
52 transition.terminated().clone(),
53 transition.truncated().clone(),
54 ))?;
55 environment.reset_completed()?;
56 }
57 rollout.finalize(advantage::GaeConfig::default())?;
58 let completion = environment.submit()?;
59 environment.wait(&completion)?;
60
61 let reward = rollout.batch().reward().read_f32()?;
62 let terminated = rollout.batch().terminated().read::<u8>()?;
63 let truncated = rollout.batch().truncated().read::<u8>()?;
64 let valid = rollout.batch().valid().read::<u8>()?;
65 let advantage = rollout.batch().advantage().read_f32()?;
66 let reward_sum = reward.iter().copied().map(f64::from).sum::<f64>();
67 let episodes = terminated
68 .iter()
69 .zip(&truncated)
70 .filter(|(terminated, truncated)| **terminated != 0 || **truncated != 0)
71 .count();
72 assert!(valid.iter().all(|value| *value == 1));
73 assert!(advantage.iter().all(|value| value.is_finite()));
74 assert_eq!(rollout.len(), HORIZON);
75 assert!(reward_sum > 0.0);
76
77 println!(
78 " result: {reward_sum:.0} reward · {episodes} completed episodes · {:.2} reward/env",
79 reward_sum / ENVIRONMENTS as f64
80 );
81 println!(" path: policy -> sample -> step -> append -> reset-done -> GAE");
82 println!(" host tensor reads during collection: 0");
83 environment.close()?;
84 Ok(())
85}
86

Run Rust tutorials from the OA checkout with cargo run --release --example <name>. Run Python with the environment containing the OA native extension. Consult each program's arguments and capability checks before running it.