CartPole rollout
Source-owned programs from the current OA SDK. Rust and Python examples follow their respective public APIs.
Rust · Python
Example
Tu_rl_cart_pole_rollout.rs
1use oa::ml::{2 RolloutBuffer, RolloutConfig, RolloutTransition, advantage, environment::Environment, policy,3};4use sdk::ml::rl::{CartPole, CartPoleConfig};56const ENVIRONMENTS: usize = 32;7const HORIZON: usize = 64;8const SEED: u64 = 20_260_716;910fn main() -> oa::Result<()> {11 let engine = oa::Engine::new()?;12 oa::Engine::set_default(engine.clone());13 let mut environment = CartPole::new(14 &engine,15 CartPoleConfig {16 environments: ENVIRONMENTS as u32,17 max_episode_steps: 500,18 seed: SEED,19 ..CartPoleConfig::default()20 },21 )?;22 let mut rollout = RolloutBuffer::new(23 &engine,24 RolloutConfig {25 time: HORIZON,26 environments: ENVIRONMENTS,27 observation_shape: vec![4],28 },29 )?;30 let policy_weight =31 oa::Matrix::new_from_f32([2, 4], &[0.0, -0.1, -4.0, -1.0, 0.0, 0.1, 4.0, 1.0])?;32 let value = oa::Matrix::new_from_f32([ENVIRONMENTS], &[0.0; ENVIRONMENTS])?;3334 println!("\nOA reinforcement learning — vectorized GPU CartPole rollout");35 println!(36 " environments: {ENVIRONMENTS} · horizon: {HORIZON} · transitions: {}",37 ENVIRONMENTS * HORIZON38 );3940 rollout.reset()?;41 for step in 0..HORIZON {42 let logits = oa::matrix::mat_mul_nt(environment.observation(), &policy_weight)?;43 let selected = policy::sample_categorical(&logits, &value, SEED + step as u64 + 1)?;44 let transition = environment.step(&selected.action)?;45 rollout.append(&RolloutTransition::new(46 transition.observation().clone(),47 selected.action,48 transition.reward().clone(),49 selected.value,50 value.clone(),51 selected.log_probability,52 transition.terminated().clone(),53 transition.truncated().clone(),54 ))?;55 environment.reset_completed()?;56 }57 rollout.finalize(advantage::GaeConfig::default())?;58 let completion = environment.submit()?;59 environment.wait(&completion)?;6061 let reward = rollout.batch().reward().read_f32()?;62 let terminated = rollout.batch().terminated().read::<u8>()?;63 let truncated = rollout.batch().truncated().read::<u8>()?;64 let valid = rollout.batch().valid().read::<u8>()?;65 let advantage = rollout.batch().advantage().read_f32()?;66 let reward_sum = reward.iter().copied().map(f64::from).sum::<f64>();67 let episodes = terminated68 .iter()69 .zip(&truncated)70 .filter(|(terminated, truncated)| **terminated != 0 || **truncated != 0)71 .count();72 assert!(valid.iter().all(|value| *value == 1));73 assert!(advantage.iter().all(|value| value.is_finite()));74 assert_eq!(rollout.len(), HORIZON);75 assert!(reward_sum > 0.0);7677 println!(78 " result: {reward_sum:.0} reward · {episodes} completed episodes · {:.2} reward/env",79 reward_sum / ENVIRONMENTS as f6480 );81 println!(" path: policy -> sample -> step -> append -> reset-done -> GAE");82 println!(" host tensor reads during collection: 0");83 environment.close()?;84 Ok(())85}86
Run Rust tutorials from the OA checkout with cargo run --release --example <name>. Run Python with the environment containing the OA native extension. Consult each program's arguments and capability checks before running it.