Image grid classification
Source-owned programs from the current OA SDK. Rust and Python examples follow their respective public APIs.
Rust · Python
Example
Tu_image_grid_classify.rs
1use std::path::{Path, PathBuf};2use std::rc::Rc;34use oa::ml::{5 GradientTape, ItTraining, ItTrainingConfig, LossMetric, Module, ModuleRegistry, ProgressBar,6 TrainingSummary,7};8use oa::plot::{Axes, Figure, FigureConfig};9use oa::ui::Ui;10use oa::{Color, Engine, Matrix, Texture, ViewerLiveCapabilities, ViewerLiveSource};1112// ── Fashion-MNIST class names ─────────────────────────────────────────────────1314const CLASSES: [&str; 10] = [15 "T-shirt", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Boot",16];17const NUM_CLASSES: usize = 10;1819// ── Title colors (donor: kSuccess / kError / kMuted) ─────────────────────────2021fn color_success() -> Color {22 Color::new(0.188, 0.820, 0.345, 1.0) // #30d15823}24fn color_error() -> Color {25 Color::new(1.000, 0.271, 0.227, 1.0) // #ff453a26}27fn color_muted() -> Color {28 Color::new(0.565, 0.565, 0.565, 1.0) // #90909029}3031// ── Fashion-MNIST IDX file loader ─────────────────────────────────────────────3233/// Raw decoded IDX dataset.34struct MnistData {35 /// Raw U8 pixel bytes, `count × 784`.36 images: Vec<u8>,37 /// Raw U8 label bytes, `count × 1`.38 labels: Vec<u8>,39 count: usize,40}4142fn read_be32(bytes: &[u8], offset: &mut usize) -> Option<u32> {43 if bytes.len().checked_sub(*offset)? < 4 {44 return None;45 }46 let b = &bytes[*offset..*offset + 4];47 *offset += 4;48 Some(u32::from_be_bytes([b[0], b[1], b[2], b[3]]))49}5051fn load_mnist_idx(dir: &Path, img_file: &str, lbl_file: &str) -> Option<MnistData> {52 let img_bytes = std::fs::read(dir.join(img_file)).ok()?;53 let lbl_bytes = std::fs::read(dir.join(lbl_file)).ok()?;5455 let mut io = 0usize;56 let mut lo = 0usize;5758 let img_magic = read_be32(&img_bytes, &mut io)?;59 if img_magic != 0x0000_0803 {60 return None;61 }62 let n = read_be32(&img_bytes, &mut io)? as usize;63 let rows = read_be32(&img_bytes, &mut io)? as usize;64 let cols = read_be32(&img_bytes, &mut io)? as usize;65 if rows != 28 || cols != 28 {66 return None;67 }6869 let lbl_magic = read_be32(&lbl_bytes, &mut lo)?;70 if lbl_magic != 0x0000_0801 {71 return None;72 }73 let lbl_n = read_be32(&lbl_bytes, &mut lo)? as usize;74 if lbl_n != n {75 return None;76 }7778 let img_expected = n * 784;79 if io + img_expected != img_bytes.len() || lo + n != lbl_bytes.len() {80 return None;81 }8283 Some(MnistData {84 images: img_bytes[io..].to_vec(),85 labels: lbl_bytes[lo..].to_vec(),86 count: n,87 })88}8990// ── MLP classifier: 784 → ReLU(128) → 10 ────────────────────────────────────9192/// Two-layer MLP. Donor: `MnistClassifier` in tuImageGridClassify.cpp.93struct MnistClassifier {94 fc1: Rc<oa::ml::nn::Linear>,95 fc2: Rc<oa::ml::nn::Linear>,96 relu: oa::ml::nn::Relu,97 registry: ModuleRegistry,98}99100impl MnistClassifier {101 fn new() -> oa::Result<Self> {102 let fc1 = Rc::new(oa::ml::nn::Linear::new_with_seed(784, 128, 1)?);103 let fc2 = Rc::new(oa::ml::nn::Linear::new_with_seed(128, NUM_CLASSES, 2)?);104 let relu = oa::ml::nn::Relu::new();105 let mut registry = ModuleRegistry::new();106 registry.register_module("fc1", fc1.clone() as Rc<dyn Module>)?;107 registry.register_module("fc2", fc2.clone() as Rc<dyn Module>)?;108 Ok(Self {109 fc1,110 fc2,111 relu,112 registry,113 })114 }115}116117impl Module for MnistClassifier {118 fn forward(&self, input: &Matrix) -> oa::Result<Matrix> {119 let h = self.fc1.forward(input)?;120 let h = self.relu.forward(&h)?;121 self.fc2.forward(&h)122 }123124 fn registry(&self) -> &ModuleRegistry {125 &self.registry126 }127}128129// ── Grid cell: one tile + prediction info ────────────────────────────────────130131struct GridCell {132 /// 28×28 RGBA8 texture.133 tile: Texture,134 actual: usize,135 predicted: usize,136 correct: bool,137}138139// ── Training + inference pipeline ────────────────────────────────────────────140141const GRID_ROWS: usize = 5;142const GRID_COLS: usize = 5;143const GRID_N: usize = GRID_ROWS * GRID_COLS; // 25144145/// Train an MLP on Fashion-MNIST, then infer on the first 25 test images.146fn train_and_predict_grid(147 engine: &Engine,148 data_dir: &Path,149 train_steps: u64,150) -> anyhow::Result<Vec<GridCell>> {151 Engine::set_default(engine.clone());152 // Load raw test data (pixel bytes needed for textures).153 let test_raw = load_mnist_idx(data_dir, "t10k-images-idx3-ubyte", "t10k-labels-idx1-ubyte")154 .ok_or_else(|| {155 anyhow::anyhow!(156 "Fashion-MNIST test set not found at {} — \157 download from https://github.com/zalandoresearch/fashion-mnist",158 data_dir.display()159 )160 })?;161162 // Load training data.163 let train_raw = load_mnist_idx(164 data_dir,165 "train-images-idx3-ubyte",166 "train-labels-idx1-ubyte",167 )168 .ok_or_else(|| {169 anyhow::anyhow!(170 "Fashion-MNIST training set not found at {}",171 data_dir.display()172 )173 })?;174175 println!(176 "Loaded {} train / {} test images",177 train_raw.count, test_raw.count178 );179180 // Build model + optimizer.181 let model = MnistClassifier::new()?;182 let params = model.all_parameters()?;183 let mut optimizer = oa::ml::AdamW::new(params, 0.001)?;184185 const BATCH: usize = 64;186 let steps_per_epoch = (train_raw.count / BATCH) as u64;187188 let mut loss_metric = LossMetric::default();189 let mut progress = ProgressBar::default();190 let mut summary = TrainingSummary::default();191 let mut training = ItTraining::new(192 engine,193 &mut optimizer,194 ItTrainingConfig {195 total_steps: train_steps,196 steps_per_epoch,197 batch_size: BATCH as u64,198 timer_name: "image_grid_classify_step".into(),199 ..ItTrainingConfig::default()200 },201 )?;202 training.add_metric(&mut loss_metric);203 training.add_callback(&mut progress);204 training.add_callback(&mut summary);205206 // Advance through the training set sequentially (deterministic, no shuffle).207 let n_train = train_raw.count;208 let mut step_offset = 0usize;209210 while training.step(211 || {212 let start = (step_offset * BATCH) % n_train.saturating_sub(1).max(BATCH);213 step_offset = step_offset.wrapping_add(1);214215 let mut x_f32 = vec![0.0f32; BATCH * 784];216 let mut y_u32 = vec![0u32; BATCH];217 for bi in 0..BATCH {218 let si = (start + bi).min(n_train - 1);219 let src = &train_raw.images[si * 784..(si + 1) * 784];220 for (j, &pixel) in src.iter().enumerate() {221 x_f32[bi * 784 + j] = pixel as f32 / 255.0;222 }223 y_u32[bi] = train_raw.labels[si] as u32;224 }225 let x = Matrix::new_from_slice([BATCH, 784], &x_f32)?;226 let y = Matrix::new_from_slice([BATCH], &y_u32)?;227 Ok((x, y))228 },229 |(x, y)| {230 let tape = GradientTape::new();231 let logits = model.forward(&x)?;232 let loss = oa::ml::loss::cross_entropy(&logits, &y)?;233 tape.backward(&loss)?;234 Ok(loss)235 },236 )? {}237238 training.finish()?;239 println!("Training done.");240241 // ── Inference on the first GRID_N test images ─────────────────────────242243 let mut x_f32 = vec![0.0f32; GRID_N * 784];244 for (i, pixel) in test_raw.images[..GRID_N * 784].iter().enumerate() {245 x_f32[i] = *pixel as f32 / 255.0;246 }247 let x_test = Matrix::new_from_slice([GRID_N, 784], &x_f32)?;248 let logits = model.forward(&x_test)?;249 let probs = oa::matrix::softmax(&logits, -1)?;250 let host_probs = probs.read_f32()?;251252 // Build grid cells.253 let mut cells = Vec::with_capacity(GRID_N);254 for i in 0..GRID_N {255 // Argmax for predicted class.256 let row = &host_probs[i * NUM_CLASSES..(i + 1) * NUM_CLASSES];257 let predicted = row258 .iter()259 .enumerate()260 .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))261 .map(|(idx, _)| idx)262 .unwrap_or(0);263 let actual = test_raw.labels[i] as usize;264265 // Build an RGBA8 grayscale texture from raw pixel bytes.266 let src = &test_raw.images[i * 784..(i + 1) * 784];267 let mut rgba = vec![0u8; 28 * 28 * 4];268 for (p, &g) in src.iter().enumerate() {269 rgba[p * 4] = g;270 rgba[p * 4 + 1] = g;271 rgba[p * 4 + 2] = g;272 rgba[p * 4 + 3] = 255;273 }274 let tile = oa::render::texture_from_rgba8(&rgba, 28, 28)?;275276 cells.push(GridCell {277 tile,278 actual,279 predicted,280 correct: predicted == actual,281 });282 }283284 let correct_count = cells.iter().filter(|c| c.correct).count();285 println!(286 "Prediction grid: {} / {} correct on the first {} test images",287 correct_count, GRID_N, GRID_N288 );289290 Ok(cells)291}292293// ── Figure population ─────────────────────────────────────────────────────────294295fn populate_figure(fig: &mut Figure, cells: &[GridCell]) {296 for (i, cell) in cells.iter().enumerate().take(GRID_N) {297 let row = (i / GRID_COLS) as i32;298 let col = (i % GRID_COLS) as i32;299 let ax: &mut Axes = fig.ax(row, col);300 ax.imshow(cell.tile.clone());301 if cell.correct {302 ax.title_color(CLASSES[cell.predicted], color_success());303 } else {304 ax.title_color(CLASSES[cell.predicted], color_error());305 ax.caption_color(CLASSES[cell.actual], color_muted());306 }307 // No grid lines or legend for image cells.308 ax.grid(false);309 ax.legend(false);310 }311}312313// ── ViewerLiveSource: drives the figure from inside a Viewer lifecycle ────────314315/// Donor: `ImageGridClassifySource` in tuImageGridClassify.cpp.316pub struct ImageGridClassifySource {317 data_dir: PathBuf,318 train_steps: u64,319 fig: Figure,320 cells: Vec<GridCell>,321}322323impl ImageGridClassifySource {324 pub fn new(data_dir: PathBuf, train_steps: u64) -> Self {325 let fig = Figure::new(FigureConfig {326 rows: GRID_ROWS as i32,327 cols: GRID_COLS as i32,328 width: 800,329 height: 800,330 h_spacing: 8,331 v_spacing: 8,332 padding: 8,333 ..Default::default()334 });335 Self {336 data_dir,337 train_steps,338 fig,339 cells: Vec::new(),340 }341 }342}343344impl ViewerLiveSource for ImageGridClassifySource {345 fn capabilities(&self) -> ViewerLiveCapabilities {346 ViewerLiveCapabilities::default()347 }348349 fn open(&mut self, engine: &Engine) -> oa::Result<()> {350 self.cells = train_and_predict_grid(engine, &self.data_dir, self.train_steps)351 .map_err(|e| oa::Error::callback(e.to_string()))?;352 populate_figure(&mut self.fig, &self.cells);353 Ok(())354 }355356 fn update(&mut self, _delta_ms: f32) -> oa::Result<()> {357 Ok(())358 }359360 fn render(&mut self, ui: &mut Ui<'_>, width: u32, height: u32) -> oa::Result<()> {361 self.fig.render_frame(width, height, ui)362 }363364 fn close(&mut self) -> oa::Result<()> {365 self.cells.clear();366 Ok(())367 }368}369370// ── CLI + main ────────────────────────────────────────────────────────────────371372fn usage(program: &str) {373 eprintln!(374 "usage: {program} [data_dir] [--save path.png] [--steps N]\n\375 \n\376 data_dir path to the Fashion-MNIST IDX files (default: ./data/fashionMnist)\n\377 --save write the 5×5 grid as a PNG instead of opening a window\n\378 --steps number of training steps (default: 2000)"379 );380}381382fn main() -> anyhow::Result<()> {383 let mut data_dir = PathBuf::from("data/fashionMnist");384 let mut save_path: Option<String> = None;385 let mut train_steps: u64 = 2000;386 let mut positional_done = false;387388 let args: Vec<String> = std::env::args().collect();389 let mut i = 1;390 while i < args.len() {391 match args[i].as_str() {392 "--save" if i + 1 < args.len() => {393 i += 1;394 save_path = Some(args[i].clone());395 }396 "--steps" if i + 1 < args.len() => {397 i += 1;398 train_steps = args[i].parse().unwrap_or(2000);399 }400 "--help" => {401 usage(&args[0]);402 return Ok(());403 }404 other if !positional_done && !other.starts_with('-') => {405 data_dir = PathBuf::from(other);406 positional_done = true;407 }408 other => {409 eprintln!("unknown argument: {other}");410 usage(&args[0]);411 std::process::exit(1);412 }413 }414 i += 1;415 }416417 println!();418 println!("╔══════════════════════════════════════════════════════════════════╗");419 println!("║ OA Tutorial — Fashion-MNIST Prediction grid (oa::plot) ║");420 println!("║ Train → Predict → display (5×5 grid, 25 test images) ║");421 println!("║ Green title = correct · Red title = wrong ║");422 println!(423 "║ Mode: {:<58}║",424 if save_path.is_some() {425 "save_to (headless PNG output)"426 } else {427 "show (interactive window)"428 }429 );430 println!("╚══════════════════════════════════════════════════════════════════╝");431 println!();432433 // ── save mode: compute-only engine, no swapchain ─────────────────────────434 if let Some(ref path) = save_path {435 let engine = Engine::new()?;436 Engine::set_default(engine.clone());437 let cells = train_and_predict_grid(&engine, &data_dir, train_steps)?;438 let mut fig = Figure::new(FigureConfig {439 rows: GRID_ROWS as i32,440 cols: GRID_COLS as i32,441 width: 800,442 height: 800,443 h_spacing: 8,444 v_spacing: 8,445 padding: 4,446 ..Default::default()447 });448 populate_figure(&mut fig, &cells);449 fig.save_to(path)?;450 println!("Saved: {path}");451 return Ok(());452 }453454 // ── show mode: Viewer lifecycle with a live figure source ─────────────────455 let mut source = ImageGridClassifySource::new(data_dir, train_steps);456 let config = oa::ViewerConfig {457 mode: oa::ViewerMode::Live,458 width: 800,459 height: 800,460 show_help: false,461 show_stats: false,462 show_timeline: false,463 ..Default::default()464 };465 let mut viewer = oa::Viewer::new(config);466 viewer.set_live_source(&mut source);467468 let window = oa::SdlWindow::new("OA — Fashion-MNIST grid", 800, 800)469 .map_err(|e| anyhow::anyhow!("SDL window: {e}"))?;470 let extensions = window471 .required_instance_extensions()472 .map_err(|e| anyhow::anyhow!("SDL extensions: {e}"))?;473 let engine = Engine::builder().instance_extensions(extensions).build()?;474 Engine::set_default(engine.clone());475 viewer.run(&engine)?;476 Ok(())477}478
Run Rust tutorials from the OA checkout with cargo run --release --example <name>. Run Python with the environment containing the OA native extension. Consult each program's arguments and capability checks before running it.