Audio command recognition
Source-owned programs from the current OA SDK. Rust and Python examples follow their respective public APIs.
Rust · Python
Example
1use std::path::PathBuf;23use oa::ml::{4 GradientTape, ItTraining, ItTrainingConfig, LossMetric, Module, ProgressBar, TrainingSummary,5};6use sdk::ml::audio;78fn arg_value(flag: &str) -> Option<String> {9 std::env::args().skip_while(|arg| arg != flag).nth(1)10}1112fn arg_usize(flag: &str, default: usize) -> usize {13 arg_value(flag)14 .and_then(|value| value.parse().ok())15 .unwrap_or(default)16}1718/// Chunked held-out evaluation pass: mean accuracy and clip-mean loss over19/// standardized host features, run eagerly between training steps.20fn evaluate(21 _engine: &oa::Engine,22 model: &impl Module,23 features: &[f32],24 labels: &[u32],25 mel_bands: usize,26 mel_frames: usize,27 chunk_clips: usize,28) -> oa::Result<(f64, f64)> {29 let clip_feature_len = mel_bands * mel_frames;30 let mut correct = 0_u64;31 let mut loss_sum = 0.0_f64;32 for begin in (0..labels.len()).step_by(chunk_clips) {33 let end = (begin + chunk_clips).min(labels.len());34 let input = oa::Matrix::new_from_slice(35 [end - begin, 1, mel_bands, mel_frames],36 &features[begin * clip_feature_len..end * clip_feature_len],37 )?;38 let target = oa::Matrix::new_from_slice([end - begin], &labels[begin..end])?;39 let logits = model.forward(&input)?;40 correct += oa::matrix::categorical_accuracy_count(&logits, &target)?.read::<u32>()?[0] as u64;41 loss_sum +=42 oa::ml::loss::cross_entropy(&logits, &target)?.read_f32()?[0] as f64 * (end - begin) as f64;43 }44 let total = labels.len().max(1) as f64;45 Ok((correct as f64 / total, loss_sum / total))46}4748fn main() -> oa::Result<()> {49 let engine = oa::Engine::new()?;50 oa::Engine::set_default(engine.clone());51 let dataset_dir =52 arg_value("--dataset").map_or_else(|| PathBuf::from("sdk/asset/audio/commands"), PathBuf::from);53 let epochs = arg_usize("--epochs", audio::TRAINING_EPOCHS);54 let batch = arg_usize("--batch", audio::BATCH_SIZE);55 let dataset = audio::SpeechCommandsDataset::load(&dataset_dir)?;56 let clip_count = dataset.clips.len();5758 println!("\n╔══════════════════════════════════════════════════════════════════╗");59 println!("║ OA Tutorial — speech commands · mel features + CNN (autograd) ║");60 println!("╚══════════════════════════════════════════════════════════════════╝\n");61 println!(62 "dataset: {} · {} clips · {} commands ({} s16kHz samples each)",63 dataset_dir.display(),64 clip_count,65 audio::COMMAND_LABELS.len(),66 audio::SAMPLES_PER_CLIP / audio::CLIP_SAMPLE_RATE as usize,67 );68 println!(69 "features: {} mel bands · fft {} · hop {} — one dispatch per batch",70 audio::MEL_BANDS,71 audio::MEL_FFT_SIZE,72 audio::MEL_HOP_SIZE,73 );7475 // Deterministic split: every 4th clip per sorted label goes to test.76 // The loader emits clips in label order, so the training indices get a77 // seeded shuffle — unshuffled windows would be single-label blocks and78 // the optimizer would collapse at every label boundary.79 let (train_idx, test_idx): (Vec<usize>, Vec<usize>) =80 (0..clip_count).partition(|&index| index % 4 != 3);81 let mut shuffled: Vec<i64> = train_idx.iter().map(|&index| index as i64).collect();82 oa::data::shuffle(&mut shuffled, audio::RNG_SEED)?;83 let train_idx: Vec<usize> = shuffled.iter().map(|&index| index as usize).collect();84 println!(85 "split: {} train · {} test\n",86 train_idx.len(),87 test_idx.len(),88 );8990 // Chunked extraction keeps each mel dispatch bounded — one call over a91 // multi-thousand-clip planar upload can exceed device memory or driver92 // watchdog limits. ItTraining::step requires an empty execution session,93 // and the prepare closure may only upload fresh inputs, so features94 // materialize to host once and windows slice host-side per step.95 let mut mel_bands = 0_usize;96 let mut mel_frames = 0_usize;97 let mut train_features_host = Vec::new();98 for chunk in train_idx.chunks(audio::MEL_CHUNK_CLIPS) {99 let features = dataset.mel_features(chunk)?;100 if mel_bands == 0 {101 // mel_features guarantees the NCHW rank-four layout.102 let shape = features.shape();103 (mel_bands, mel_frames) = (shape[2], shape[3]);104 }105 train_features_host.extend_from_slice(&features.read_f32()?);106 }107 let clip_feature_len = mel_bands * mel_frames;108 let train_labels_host = dataset.labels(&train_idx)?.read::<u32>()?;109 // Standardize with training-set statistics — the host-side equivalent of110 // the TF tutorial's Normalization layer.111 let (mean, inv_std) = audio::standardization(&train_features_host);112 audio::standardize(&mut train_features_host, mean, inv_std);113 // Held-out split materializes upfront — the per-epoch validation pass runs114 // eagerly between training steps.115 let mut test_features_host = Vec::new();116 for chunk in test_idx.chunks(audio::MEL_CHUNK_CLIPS) {117 let features = dataset.mel_features(chunk)?;118 test_features_host.extend_from_slice(&features.read_f32()?);119 }120 audio::standardize(&mut test_features_host, mean, inv_std);121 let test_labels_host: Vec<u32> = test_idx122 .iter()123 .map(|&index| dataset.clips[index].label)124 .collect();125 let model = audio::commands_cnn(mel_bands, mel_frames, audio::COMMAND_LABELS.len())?;126 model.train(true);127 // Complete asynchronous parameter initialization before isolated capture.128 engine.checkpoint()?.wait()?;129 // TF's Adam default — 1e-2 diverges on the full dataset's early batches.130 let learning_rate = 0.001;131 let mut optimizer = oa::ml::AdamW::new(model.all_parameters()?, learning_rate)?;132133 println!(134 "Model: Conv2d(1→8,3×3)→ReLU→MaxPool→Conv2d(8→16,3×3)→ReLU→MaxPool→Linear(→64)→Dropout→Linear(→8)"135 );136 println!(137 "params: {} Optimizer: AdamW(lr={learning_rate})\n",138 model.num_parameters()?139 );140141 // Full-width windows only — a fixed batch shape keeps the captured142 // training program stable; the tail remainder is dropped.143 let windows = (train_idx.len() / batch).max(1);144 // The canonical recipe counts data passes like TensorFlow: epochs ×145 // windows-per-pass gives the total step count. --steps overrides it.146 let steps = arg_usize("--steps", epochs * windows);147 let mut loss_metric = LossMetric::default();148 let mut progress = ProgressBar::default();149 let mut summary_callback = TrainingSummary::default();150 let mut training = ItTraining::new(151 &engine,152 &mut optimizer,153 ItTrainingConfig {154 total_steps: steps as u64,155 steps_per_epoch: windows as u64,156 batch_size: batch as u64,157 source_unit: "clip".into(),158 timer_name: "commands_training_step".into(),159 enable_gpu_timing: true,160 ..ItTrainingConfig::default()161 },162 )?;163 training.add_metric(&mut loss_metric);164 training.add_callback(&mut progress);165 training.add_callback(&mut summary_callback);166167 let mut initial_loss = 0.0_f32;168 // The per-step accuracy count is recorded as an extra program output and169 // read back after each step — the same readback the loss metric uses.170 let step_correct = std::cell::RefCell::new(Option::<oa::Matrix>::None);171 let mut epoch_correct = 0_u64;172 let mut epoch_clips = 0_u64;173 for step in 0..steps {174 training.record_source_units(batch as u64);175 // Reshuffle the window order each data pass, as TF's176 // dataset.shuffle does per epoch.177 let mut window_order: Vec<i64> = (0..windows as i64).collect();178 oa::data::shuffle(179 &mut window_order,180 audio::RNG_SEED.wrapping_add((step / windows) as u64),181 )?;182 assert!(training.step(183 || {184 let begin = (window_order[step % windows] as usize * batch)185 .min(train_idx.len().saturating_sub(batch));186 let end = begin + batch.min(train_idx.len());187 let input = oa::Matrix::new_from_slice(188 [end - begin, 1, mel_bands, mel_frames],189 &train_features_host[begin * clip_feature_len..end * clip_feature_len],190 )?;191 let target = oa::Matrix::new_from_slice([end - begin], &train_labels_host[begin..end])?;192 Ok((input, target))193 },194 |(input, target)| {195 let tape = GradientTape::new();196 let logits = model.forward(&input)?;197 let loss = oa::ml::loss::cross_entropy(&logits, &target)?;198 tape.backward(&loss)?;199 *step_correct.borrow_mut() =200 Some(oa::matrix::categorical_accuracy_count(&logits, &target)?);201 Ok(loss)202 },203 )?);204 if let Some(count) = &*step_correct.borrow() {205 epoch_correct += count.read::<u32>()?[0] as u64;206 }207 epoch_clips += batch.min(train_idx.len()) as u64;208 let snapshot = training.snapshot();209 if snapshot.step_count() == 1 {210 initial_loss = snapshot.last_loss().unwrap_or_default();211 }212 if snapshot.is_epoch_boundary() {213 model.eval();214 let (val_accuracy, val_loss) = evaluate(215 &engine,216 &model,217 &test_features_host,218 &test_labels_host,219 mel_bands,220 mel_frames,221 audio::MEL_CHUNK_CLIPS,222 )?;223 model.train(true);224 let seconds = snapshot.epoch_elapsed().as_secs_f64();225 println!(226 " {}/{} — {seconds:.1}s {:.0} ms/step · accuracy: {:.4} · loss: {:.4} · val_accuracy: {val_accuracy:.4} · val_loss: {val_loss:.4}",227 snapshot.step_in_epoch(),228 snapshot.steps_in_epoch(),229 seconds * 1e3 / snapshot.steps_in_epoch().max(1) as f64,230 epoch_correct as f64 / epoch_clips.max(1) as f64,231 snapshot.epoch_mean_loss(),232 );233 epoch_correct = 0;234 epoch_clips = 0;235 }236 }237 let program_diagnostics = training238 .training_program()239 .map(|program| program.diagnostics());240 let result = training.finish()?;241 let final_training_loss = result.last_loss().unwrap_or_default();242243 // Final held-out evaluation through the same chunked pass the per-epoch244 // validation used.245 model.eval();246 let (accuracy, evaluation_loss) = evaluate(247 &engine,248 &model,249 &test_features_host,250 &test_labels_host,251 mel_bands,252 mel_frames,253 audio::MEL_CHUNK_CLIPS,254 )?;255256 println!("\nEvaluation:");257 println!(258 " Random-loss baseline ln({}) = {:.4}",259 audio::COMMAND_LABELS.len(),260 (audio::COMMAND_LABELS.len() as f64).ln()261 );262 println!(" test accuracy: {:.1}%", accuracy * 100.0);263 println!(" post-update evaluation loss: {evaluation_loss:.6}");264 if let Some(diagnostics) = program_diagnostics {265 println!(266 "program: {} nodes · {} recording · {} cache hits · {} uploads",267 diagnostics.node_count(),268 diagnostics.command_recording_count(),269 diagnostics.command_cache_hit_count(),270 diagnostics.input_upload_count(),271 );272 }273274 assert!(initial_loss > 1.0);275 assert!(final_training_loss < initial_loss);276 // The recorded oracles describe the canonical bundled-subset recipe only.277 let canonical = arg_value("--dataset").is_none()278 && arg_value("--steps").is_none()279 && epochs == audio::TRAINING_EPOCHS280 && batch == audio::BATCH_SIZE;281 if canonical {282 assert!((final_training_loss - audio::COMMANDS_FINAL_LOSS).abs() < 0.01);283 assert!(accuracy >= f64::from(audio::COMMANDS_TEST_ACCURACY));284 }285286 let checkpoint = oa::Path::temp().join("oars_commands").with_extension("oam");287 oa::ml::save_checkpoint(&checkpoint, &model, &optimizer)?;288 let reloaded = audio::commands_cnn(mel_bands, mel_frames, audio::COMMAND_LABELS.len())?;289 reloaded.eval();290 let mut reloaded_optimizer = oa::ml::AdamW::new(reloaded.all_parameters()?, learning_rate)?;291 oa::ml::load_checkpoint(&checkpoint, &reloaded, &mut reloaded_optimizer)?;292 assert_eq!(reloaded_optimizer.step_count(), optimizer.step_count());293 assert_eq!(294 evaluate(295 &engine,296 &reloaded,297 &test_features_host,298 &test_labels_host,299 mel_bands,300 mel_frames,301 audio::MEL_CHUNK_CLIPS,302 )?303 .0,304 accuracy305 );306 oa::Filesystem::remove_file(&checkpoint).expect("remove completed commands checkpoint");307 Ok(())308}309
Run Rust tutorials from the OA checkout with cargo run --release --example <name>. Run Python with the environment containing the OA native extension. Consult each program's arguments and capability checks before running it.