Realm

Audio command recognition

Source-owned programs from the current OA SDK. Rust and Python examples follow their respective public APIs.

Rust · Python

Example

1use std::path::PathBuf;
2
3use oa::ml::{
4 GradientTape, ItTraining, ItTrainingConfig, LossMetric, Module, ProgressBar, TrainingSummary,
5};
6use sdk::ml::audio;
7
8fn arg_value(flag: &str) -> Option<String> {
9 std::env::args().skip_while(|arg| arg != flag).nth(1)
10}
11
12fn arg_usize(flag: &str, default: usize) -> usize {
13 arg_value(flag)
14 .and_then(|value| value.parse().ok())
15 .unwrap_or(default)
16}
17
18/// Chunked held-out evaluation pass: mean accuracy and clip-mean loss over
19/// standardized host features, run eagerly between training steps.
20fn evaluate(
21 _engine: &oa::Engine,
22 model: &impl Module,
23 features: &[f32],
24 labels: &[u32],
25 mel_bands: usize,
26 mel_frames: usize,
27 chunk_clips: usize,
28) -> oa::Result<(f64, f64)> {
29 let clip_feature_len = mel_bands * mel_frames;
30 let mut correct = 0_u64;
31 let mut loss_sum = 0.0_f64;
32 for begin in (0..labels.len()).step_by(chunk_clips) {
33 let end = (begin + chunk_clips).min(labels.len());
34 let input = oa::Matrix::new_from_slice(
35 [end - begin, 1, mel_bands, mel_frames],
36 &features[begin * clip_feature_len..end * clip_feature_len],
37 )?;
38 let target = oa::Matrix::new_from_slice([end - begin], &labels[begin..end])?;
39 let logits = model.forward(&input)?;
40 correct += oa::matrix::categorical_accuracy_count(&logits, &target)?.read::<u32>()?[0] as u64;
41 loss_sum +=
42 oa::ml::loss::cross_entropy(&logits, &target)?.read_f32()?[0] as f64 * (end - begin) as f64;
43 }
44 let total = labels.len().max(1) as f64;
45 Ok((correct as f64 / total, loss_sum / total))
46}
47
48fn main() -> oa::Result<()> {
49 let engine = oa::Engine::new()?;
50 oa::Engine::set_default(engine.clone());
51 let dataset_dir =
52 arg_value("--dataset").map_or_else(|| PathBuf::from("sdk/asset/audio/commands"), PathBuf::from);
53 let epochs = arg_usize("--epochs", audio::TRAINING_EPOCHS);
54 let batch = arg_usize("--batch", audio::BATCH_SIZE);
55 let dataset = audio::SpeechCommandsDataset::load(&dataset_dir)?;
56 let clip_count = dataset.clips.len();
57
58 println!("\n╔══════════════════════════════════════════════════════════════════╗");
59 println!("║ OA Tutorial — speech commands · mel features + CNN (autograd) ║");
60 println!("╚══════════════════════════════════════════════════════════════════╝\n");
61 println!(
62 "dataset: {} · {} clips · {} commands ({} s16kHz samples each)",
63 dataset_dir.display(),
64 clip_count,
65 audio::COMMAND_LABELS.len(),
66 audio::SAMPLES_PER_CLIP / audio::CLIP_SAMPLE_RATE as usize,
67 );
68 println!(
69 "features: {} mel bands · fft {} · hop {} — one dispatch per batch",
70 audio::MEL_BANDS,
71 audio::MEL_FFT_SIZE,
72 audio::MEL_HOP_SIZE,
73 );
74
75 // Deterministic split: every 4th clip per sorted label goes to test.
76 // The loader emits clips in label order, so the training indices get a
77 // seeded shuffle — unshuffled windows would be single-label blocks and
78 // the optimizer would collapse at every label boundary.
79 let (train_idx, test_idx): (Vec<usize>, Vec<usize>) =
80 (0..clip_count).partition(|&index| index % 4 != 3);
81 let mut shuffled: Vec<i64> = train_idx.iter().map(|&index| index as i64).collect();
82 oa::data::shuffle(&mut shuffled, audio::RNG_SEED)?;
83 let train_idx: Vec<usize> = shuffled.iter().map(|&index| index as usize).collect();
84 println!(
85 "split: {} train · {} test\n",
86 train_idx.len(),
87 test_idx.len(),
88 );
89
90 // Chunked extraction keeps each mel dispatch bounded — one call over a
91 // multi-thousand-clip planar upload can exceed device memory or driver
92 // watchdog limits. ItTraining::step requires an empty execution session,
93 // and the prepare closure may only upload fresh inputs, so features
94 // materialize to host once and windows slice host-side per step.
95 let mut mel_bands = 0_usize;
96 let mut mel_frames = 0_usize;
97 let mut train_features_host = Vec::new();
98 for chunk in train_idx.chunks(audio::MEL_CHUNK_CLIPS) {
99 let features = dataset.mel_features(chunk)?;
100 if mel_bands == 0 {
101 // mel_features guarantees the NCHW rank-four layout.
102 let shape = features.shape();
103 (mel_bands, mel_frames) = (shape[2], shape[3]);
104 }
105 train_features_host.extend_from_slice(&features.read_f32()?);
106 }
107 let clip_feature_len = mel_bands * mel_frames;
108 let train_labels_host = dataset.labels(&train_idx)?.read::<u32>()?;
109 // Standardize with training-set statistics — the host-side equivalent of
110 // the TF tutorial's Normalization layer.
111 let (mean, inv_std) = audio::standardization(&train_features_host);
112 audio::standardize(&mut train_features_host, mean, inv_std);
113 // Held-out split materializes upfront — the per-epoch validation pass runs
114 // eagerly between training steps.
115 let mut test_features_host = Vec::new();
116 for chunk in test_idx.chunks(audio::MEL_CHUNK_CLIPS) {
117 let features = dataset.mel_features(chunk)?;
118 test_features_host.extend_from_slice(&features.read_f32()?);
119 }
120 audio::standardize(&mut test_features_host, mean, inv_std);
121 let test_labels_host: Vec<u32> = test_idx
122 .iter()
123 .map(|&index| dataset.clips[index].label)
124 .collect();
125 let model = audio::commands_cnn(mel_bands, mel_frames, audio::COMMAND_LABELS.len())?;
126 model.train(true);
127 // Complete asynchronous parameter initialization before isolated capture.
128 engine.checkpoint()?.wait()?;
129 // TF's Adam default — 1e-2 diverges on the full dataset's early batches.
130 let learning_rate = 0.001;
131 let mut optimizer = oa::ml::AdamW::new(model.all_parameters()?, learning_rate)?;
132
133 println!(
134 "Model: Conv2d(1→8,3×3)→ReLU→MaxPool→Conv2d(8→16,3×3)→ReLU→MaxPool→Linear(→64)→Dropout→Linear(→8)"
135 );
136 println!(
137 "params: {} Optimizer: AdamW(lr={learning_rate})\n",
138 model.num_parameters()?
139 );
140
141 // Full-width windows only — a fixed batch shape keeps the captured
142 // training program stable; the tail remainder is dropped.
143 let windows = (train_idx.len() / batch).max(1);
144 // The canonical recipe counts data passes like TensorFlow: epochs ×
145 // windows-per-pass gives the total step count. --steps overrides it.
146 let steps = arg_usize("--steps", epochs * windows);
147 let mut loss_metric = LossMetric::default();
148 let mut progress = ProgressBar::default();
149 let mut summary_callback = TrainingSummary::default();
150 let mut training = ItTraining::new(
151 &engine,
152 &mut optimizer,
153 ItTrainingConfig {
154 total_steps: steps as u64,
155 steps_per_epoch: windows as u64,
156 batch_size: batch as u64,
157 source_unit: "clip".into(),
158 timer_name: "commands_training_step".into(),
159 enable_gpu_timing: true,
160 ..ItTrainingConfig::default()
161 },
162 )?;
163 training.add_metric(&mut loss_metric);
164 training.add_callback(&mut progress);
165 training.add_callback(&mut summary_callback);
166
167 let mut initial_loss = 0.0_f32;
168 // The per-step accuracy count is recorded as an extra program output and
169 // read back after each step — the same readback the loss metric uses.
170 let step_correct = std::cell::RefCell::new(Option::<oa::Matrix>::None);
171 let mut epoch_correct = 0_u64;
172 let mut epoch_clips = 0_u64;
173 for step in 0..steps {
174 training.record_source_units(batch as u64);
175 // Reshuffle the window order each data pass, as TF's
176 // dataset.shuffle does per epoch.
177 let mut window_order: Vec<i64> = (0..windows as i64).collect();
178 oa::data::shuffle(
179 &mut window_order,
180 audio::RNG_SEED.wrapping_add((step / windows) as u64),
181 )?;
182 assert!(training.step(
183 || {
184 let begin = (window_order[step % windows] as usize * batch)
185 .min(train_idx.len().saturating_sub(batch));
186 let end = begin + batch.min(train_idx.len());
187 let input = oa::Matrix::new_from_slice(
188 [end - begin, 1, mel_bands, mel_frames],
189 &train_features_host[begin * clip_feature_len..end * clip_feature_len],
190 )?;
191 let target = oa::Matrix::new_from_slice([end - begin], &train_labels_host[begin..end])?;
192 Ok((input, target))
193 },
194 |(input, target)| {
195 let tape = GradientTape::new();
196 let logits = model.forward(&input)?;
197 let loss = oa::ml::loss::cross_entropy(&logits, &target)?;
198 tape.backward(&loss)?;
199 *step_correct.borrow_mut() =
200 Some(oa::matrix::categorical_accuracy_count(&logits, &target)?);
201 Ok(loss)
202 },
203 )?);
204 if let Some(count) = &*step_correct.borrow() {
205 epoch_correct += count.read::<u32>()?[0] as u64;
206 }
207 epoch_clips += batch.min(train_idx.len()) as u64;
208 let snapshot = training.snapshot();
209 if snapshot.step_count() == 1 {
210 initial_loss = snapshot.last_loss().unwrap_or_default();
211 }
212 if snapshot.is_epoch_boundary() {
213 model.eval();
214 let (val_accuracy, val_loss) = evaluate(
215 &engine,
216 &model,
217 &test_features_host,
218 &test_labels_host,
219 mel_bands,
220 mel_frames,
221 audio::MEL_CHUNK_CLIPS,
222 )?;
223 model.train(true);
224 let seconds = snapshot.epoch_elapsed().as_secs_f64();
225 println!(
226 " {}/{} — {seconds:.1}s {:.0} ms/step · accuracy: {:.4} · loss: {:.4} · val_accuracy: {val_accuracy:.4} · val_loss: {val_loss:.4}",
227 snapshot.step_in_epoch(),
228 snapshot.steps_in_epoch(),
229 seconds * 1e3 / snapshot.steps_in_epoch().max(1) as f64,
230 epoch_correct as f64 / epoch_clips.max(1) as f64,
231 snapshot.epoch_mean_loss(),
232 );
233 epoch_correct = 0;
234 epoch_clips = 0;
235 }
236 }
237 let program_diagnostics = training
238 .training_program()
239 .map(|program| program.diagnostics());
240 let result = training.finish()?;
241 let final_training_loss = result.last_loss().unwrap_or_default();
242
243 // Final held-out evaluation through the same chunked pass the per-epoch
244 // validation used.
245 model.eval();
246 let (accuracy, evaluation_loss) = evaluate(
247 &engine,
248 &model,
249 &test_features_host,
250 &test_labels_host,
251 mel_bands,
252 mel_frames,
253 audio::MEL_CHUNK_CLIPS,
254 )?;
255
256 println!("\nEvaluation:");
257 println!(
258 " Random-loss baseline ln({}) = {:.4}",
259 audio::COMMAND_LABELS.len(),
260 (audio::COMMAND_LABELS.len() as f64).ln()
261 );
262 println!(" test accuracy: {:.1}%", accuracy * 100.0);
263 println!(" post-update evaluation loss: {evaluation_loss:.6}");
264 if let Some(diagnostics) = program_diagnostics {
265 println!(
266 "program: {} nodes · {} recording · {} cache hits · {} uploads",
267 diagnostics.node_count(),
268 diagnostics.command_recording_count(),
269 diagnostics.command_cache_hit_count(),
270 diagnostics.input_upload_count(),
271 );
272 }
273
274 assert!(initial_loss > 1.0);
275 assert!(final_training_loss < initial_loss);
276 // The recorded oracles describe the canonical bundled-subset recipe only.
277 let canonical = arg_value("--dataset").is_none()
278 && arg_value("--steps").is_none()
279 && epochs == audio::TRAINING_EPOCHS
280 && batch == audio::BATCH_SIZE;
281 if canonical {
282 assert!((final_training_loss - audio::COMMANDS_FINAL_LOSS).abs() < 0.01);
283 assert!(accuracy >= f64::from(audio::COMMANDS_TEST_ACCURACY));
284 }
285
286 let checkpoint = oa::Path::temp().join("oars_commands").with_extension("oam");
287 oa::ml::save_checkpoint(&checkpoint, &model, &optimizer)?;
288 let reloaded = audio::commands_cnn(mel_bands, mel_frames, audio::COMMAND_LABELS.len())?;
289 reloaded.eval();
290 let mut reloaded_optimizer = oa::ml::AdamW::new(reloaded.all_parameters()?, learning_rate)?;
291 oa::ml::load_checkpoint(&checkpoint, &reloaded, &mut reloaded_optimizer)?;
292 assert_eq!(reloaded_optimizer.step_count(), optimizer.step_count());
293 assert_eq!(
294 evaluate(
295 &engine,
296 &reloaded,
297 &test_features_host,
298 &test_labels_host,
299 mel_bands,
300 mel_frames,
301 audio::MEL_CHUNK_CLIPS,
302 )?
303 .0,
304 accuracy
305 );
306 oa::Filesystem::remove_file(&checkpoint).expect("remove completed commands checkpoint");
307 Ok(())
308}
309

Run Rust tutorials from the OA checkout with cargo run --release --example <name>. Run Python with the environment containing the OA native extension. Consult each program's arguments and capability checks before running it.