Realm

Image grid classification

Source-owned programs from the current OA SDK. Rust and Python examples follow their respective public APIs.

Rust · Python

Example

Tu_image_grid_classify.rs

1use std::path::{Path, PathBuf};
2use std::rc::Rc;
3
4use oa::ml::{
5 GradientTape, ItTraining, ItTrainingConfig, LossMetric, Module, ModuleRegistry, ProgressBar,
6 TrainingSummary,
7};
8use oa::plot::{Axes, Figure, FigureConfig};
9use oa::ui::Ui;
10use oa::{Color, Engine, Matrix, Texture, ViewerLiveCapabilities, ViewerLiveSource};
11
12// ── Fashion-MNIST class names ─────────────────────────────────────────────────
13
14const CLASSES: [&str; 10] = [
15 "T-shirt", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Boot",
16];
17const NUM_CLASSES: usize = 10;
18
19// ── Title colors (donor: kSuccess / kError / kMuted) ─────────────────────────
20
21fn color_success() -> Color {
22 Color::new(0.188, 0.820, 0.345, 1.0) // #30d158
23}
24fn color_error() -> Color {
25 Color::new(1.000, 0.271, 0.227, 1.0) // #ff453a
26}
27fn color_muted() -> Color {
28 Color::new(0.565, 0.565, 0.565, 1.0) // #909090
29}
30
31// ── Fashion-MNIST IDX file loader ─────────────────────────────────────────────
32
33/// Raw decoded IDX dataset.
34struct MnistData {
35 /// Raw U8 pixel bytes, `count × 784`.
36 images: Vec<u8>,
37 /// Raw U8 label bytes, `count × 1`.
38 labels: Vec<u8>,
39 count: usize,
40}
41
42fn read_be32(bytes: &[u8], offset: &mut usize) -> Option<u32> {
43 if bytes.len().checked_sub(*offset)? < 4 {
44 return None;
45 }
46 let b = &bytes[*offset..*offset + 4];
47 *offset += 4;
48 Some(u32::from_be_bytes([b[0], b[1], b[2], b[3]]))
49}
50
51fn load_mnist_idx(dir: &Path, img_file: &str, lbl_file: &str) -> Option<MnistData> {
52 let img_bytes = std::fs::read(dir.join(img_file)).ok()?;
53 let lbl_bytes = std::fs::read(dir.join(lbl_file)).ok()?;
54
55 let mut io = 0usize;
56 let mut lo = 0usize;
57
58 let img_magic = read_be32(&img_bytes, &mut io)?;
59 if img_magic != 0x0000_0803 {
60 return None;
61 }
62 let n = read_be32(&img_bytes, &mut io)? as usize;
63 let rows = read_be32(&img_bytes, &mut io)? as usize;
64 let cols = read_be32(&img_bytes, &mut io)? as usize;
65 if rows != 28 || cols != 28 {
66 return None;
67 }
68
69 let lbl_magic = read_be32(&lbl_bytes, &mut lo)?;
70 if lbl_magic != 0x0000_0801 {
71 return None;
72 }
73 let lbl_n = read_be32(&lbl_bytes, &mut lo)? as usize;
74 if lbl_n != n {
75 return None;
76 }
77
78 let img_expected = n * 784;
79 if io + img_expected != img_bytes.len() || lo + n != lbl_bytes.len() {
80 return None;
81 }
82
83 Some(MnistData {
84 images: img_bytes[io..].to_vec(),
85 labels: lbl_bytes[lo..].to_vec(),
86 count: n,
87 })
88}
89
90// ── MLP classifier: 784 → ReLU(128) → 10 ────────────────────────────────────
91
92/// Two-layer MLP. Donor: `MnistClassifier` in tuImageGridClassify.cpp.
93struct MnistClassifier {
94 fc1: Rc<oa::ml::nn::Linear>,
95 fc2: Rc<oa::ml::nn::Linear>,
96 relu: oa::ml::nn::Relu,
97 registry: ModuleRegistry,
98}
99
100impl MnistClassifier {
101 fn new() -> oa::Result<Self> {
102 let fc1 = Rc::new(oa::ml::nn::Linear::new_with_seed(784, 128, 1)?);
103 let fc2 = Rc::new(oa::ml::nn::Linear::new_with_seed(128, NUM_CLASSES, 2)?);
104 let relu = oa::ml::nn::Relu::new();
105 let mut registry = ModuleRegistry::new();
106 registry.register_module("fc1", fc1.clone() as Rc<dyn Module>)?;
107 registry.register_module("fc2", fc2.clone() as Rc<dyn Module>)?;
108 Ok(Self {
109 fc1,
110 fc2,
111 relu,
112 registry,
113 })
114 }
115}
116
117impl Module for MnistClassifier {
118 fn forward(&self, input: &Matrix) -> oa::Result<Matrix> {
119 let h = self.fc1.forward(input)?;
120 let h = self.relu.forward(&h)?;
121 self.fc2.forward(&h)
122 }
123
124 fn registry(&self) -> &ModuleRegistry {
125 &self.registry
126 }
127}
128
129// ── Grid cell: one tile + prediction info ────────────────────────────────────
130
131struct GridCell {
132 /// 28×28 RGBA8 texture.
133 tile: Texture,
134 actual: usize,
135 predicted: usize,
136 correct: bool,
137}
138
139// ── Training + inference pipeline ────────────────────────────────────────────
140
141const GRID_ROWS: usize = 5;
142const GRID_COLS: usize = 5;
143const GRID_N: usize = GRID_ROWS * GRID_COLS; // 25
144
145/// Train an MLP on Fashion-MNIST, then infer on the first 25 test images.
146fn train_and_predict_grid(
147 engine: &Engine,
148 data_dir: &Path,
149 train_steps: u64,
150) -> anyhow::Result<Vec<GridCell>> {
151 Engine::set_default(engine.clone());
152 // Load raw test data (pixel bytes needed for textures).
153 let test_raw = load_mnist_idx(data_dir, "t10k-images-idx3-ubyte", "t10k-labels-idx1-ubyte")
154 .ok_or_else(|| {
155 anyhow::anyhow!(
156 "Fashion-MNIST test set not found at {} — \
157 download from https://github.com/zalandoresearch/fashion-mnist",
158 data_dir.display()
159 )
160 })?;
161
162 // Load training data.
163 let train_raw = load_mnist_idx(
164 data_dir,
165 "train-images-idx3-ubyte",
166 "train-labels-idx1-ubyte",
167 )
168 .ok_or_else(|| {
169 anyhow::anyhow!(
170 "Fashion-MNIST training set not found at {}",
171 data_dir.display()
172 )
173 })?;
174
175 println!(
176 "Loaded {} train / {} test images",
177 train_raw.count, test_raw.count
178 );
179
180 // Build model + optimizer.
181 let model = MnistClassifier::new()?;
182 let params = model.all_parameters()?;
183 let mut optimizer = oa::ml::AdamW::new(params, 0.001)?;
184
185 const BATCH: usize = 64;
186 let steps_per_epoch = (train_raw.count / BATCH) as u64;
187
188 let mut loss_metric = LossMetric::default();
189 let mut progress = ProgressBar::default();
190 let mut summary = TrainingSummary::default();
191 let mut training = ItTraining::new(
192 engine,
193 &mut optimizer,
194 ItTrainingConfig {
195 total_steps: train_steps,
196 steps_per_epoch,
197 batch_size: BATCH as u64,
198 timer_name: "image_grid_classify_step".into(),
199 ..ItTrainingConfig::default()
200 },
201 )?;
202 training.add_metric(&mut loss_metric);
203 training.add_callback(&mut progress);
204 training.add_callback(&mut summary);
205
206 // Advance through the training set sequentially (deterministic, no shuffle).
207 let n_train = train_raw.count;
208 let mut step_offset = 0usize;
209
210 while training.step(
211 || {
212 let start = (step_offset * BATCH) % n_train.saturating_sub(1).max(BATCH);
213 step_offset = step_offset.wrapping_add(1);
214
215 let mut x_f32 = vec![0.0f32; BATCH * 784];
216 let mut y_u32 = vec![0u32; BATCH];
217 for bi in 0..BATCH {
218 let si = (start + bi).min(n_train - 1);
219 let src = &train_raw.images[si * 784..(si + 1) * 784];
220 for (j, &pixel) in src.iter().enumerate() {
221 x_f32[bi * 784 + j] = pixel as f32 / 255.0;
222 }
223 y_u32[bi] = train_raw.labels[si] as u32;
224 }
225 let x = Matrix::new_from_slice([BATCH, 784], &x_f32)?;
226 let y = Matrix::new_from_slice([BATCH], &y_u32)?;
227 Ok((x, y))
228 },
229 |(x, y)| {
230 let tape = GradientTape::new();
231 let logits = model.forward(&x)?;
232 let loss = oa::ml::loss::cross_entropy(&logits, &y)?;
233 tape.backward(&loss)?;
234 Ok(loss)
235 },
236 )? {}
237
238 training.finish()?;
239 println!("Training done.");
240
241 // ── Inference on the first GRID_N test images ─────────────────────────
242
243 let mut x_f32 = vec![0.0f32; GRID_N * 784];
244 for (i, pixel) in test_raw.images[..GRID_N * 784].iter().enumerate() {
245 x_f32[i] = *pixel as f32 / 255.0;
246 }
247 let x_test = Matrix::new_from_slice([GRID_N, 784], &x_f32)?;
248 let logits = model.forward(&x_test)?;
249 let probs = oa::matrix::softmax(&logits, -1)?;
250 let host_probs = probs.read_f32()?;
251
252 // Build grid cells.
253 let mut cells = Vec::with_capacity(GRID_N);
254 for i in 0..GRID_N {
255 // Argmax for predicted class.
256 let row = &host_probs[i * NUM_CLASSES..(i + 1) * NUM_CLASSES];
257 let predicted = row
258 .iter()
259 .enumerate()
260 .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
261 .map(|(idx, _)| idx)
262 .unwrap_or(0);
263 let actual = test_raw.labels[i] as usize;
264
265 // Build an RGBA8 grayscale texture from raw pixel bytes.
266 let src = &test_raw.images[i * 784..(i + 1) * 784];
267 let mut rgba = vec![0u8; 28 * 28 * 4];
268 for (p, &g) in src.iter().enumerate() {
269 rgba[p * 4] = g;
270 rgba[p * 4 + 1] = g;
271 rgba[p * 4 + 2] = g;
272 rgba[p * 4 + 3] = 255;
273 }
274 let tile = oa::render::texture_from_rgba8(&rgba, 28, 28)?;
275
276 cells.push(GridCell {
277 tile,
278 actual,
279 predicted,
280 correct: predicted == actual,
281 });
282 }
283
284 let correct_count = cells.iter().filter(|c| c.correct).count();
285 println!(
286 "Prediction grid: {} / {} correct on the first {} test images",
287 correct_count, GRID_N, GRID_N
288 );
289
290 Ok(cells)
291}
292
293// ── Figure population ─────────────────────────────────────────────────────────
294
295fn populate_figure(fig: &mut Figure, cells: &[GridCell]) {
296 for (i, cell) in cells.iter().enumerate().take(GRID_N) {
297 let row = (i / GRID_COLS) as i32;
298 let col = (i % GRID_COLS) as i32;
299 let ax: &mut Axes = fig.ax(row, col);
300 ax.imshow(cell.tile.clone());
301 if cell.correct {
302 ax.title_color(CLASSES[cell.predicted], color_success());
303 } else {
304 ax.title_color(CLASSES[cell.predicted], color_error());
305 ax.caption_color(CLASSES[cell.actual], color_muted());
306 }
307 // No grid lines or legend for image cells.
308 ax.grid(false);
309 ax.legend(false);
310 }
311}
312
313// ── ViewerLiveSource: drives the figure from inside a Viewer lifecycle ────────
314
315/// Donor: `ImageGridClassifySource` in tuImageGridClassify.cpp.
316pub struct ImageGridClassifySource {
317 data_dir: PathBuf,
318 train_steps: u64,
319 fig: Figure,
320 cells: Vec<GridCell>,
321}
322
323impl ImageGridClassifySource {
324 pub fn new(data_dir: PathBuf, train_steps: u64) -> Self {
325 let fig = Figure::new(FigureConfig {
326 rows: GRID_ROWS as i32,
327 cols: GRID_COLS as i32,
328 width: 800,
329 height: 800,
330 h_spacing: 8,
331 v_spacing: 8,
332 padding: 8,
333 ..Default::default()
334 });
335 Self {
336 data_dir,
337 train_steps,
338 fig,
339 cells: Vec::new(),
340 }
341 }
342}
343
344impl ViewerLiveSource for ImageGridClassifySource {
345 fn capabilities(&self) -> ViewerLiveCapabilities {
346 ViewerLiveCapabilities::default()
347 }
348
349 fn open(&mut self, engine: &Engine) -> oa::Result<()> {
350 self.cells = train_and_predict_grid(engine, &self.data_dir, self.train_steps)
351 .map_err(|e| oa::Error::callback(e.to_string()))?;
352 populate_figure(&mut self.fig, &self.cells);
353 Ok(())
354 }
355
356 fn update(&mut self, _delta_ms: f32) -> oa::Result<()> {
357 Ok(())
358 }
359
360 fn render(&mut self, ui: &mut Ui<'_>, width: u32, height: u32) -> oa::Result<()> {
361 self.fig.render_frame(width, height, ui)
362 }
363
364 fn close(&mut self) -> oa::Result<()> {
365 self.cells.clear();
366 Ok(())
367 }
368}
369
370// ── CLI + main ────────────────────────────────────────────────────────────────
371
372fn usage(program: &str) {
373 eprintln!(
374 "usage: {program} [data_dir] [--save path.png] [--steps N]\n\
375 \n\
376 data_dir path to the Fashion-MNIST IDX files (default: ./data/fashionMnist)\n\
377 --save write the 5×5 grid as a PNG instead of opening a window\n\
378 --steps number of training steps (default: 2000)"
379 );
380}
381
382fn main() -> anyhow::Result<()> {
383 let mut data_dir = PathBuf::from("data/fashionMnist");
384 let mut save_path: Option<String> = None;
385 let mut train_steps: u64 = 2000;
386 let mut positional_done = false;
387
388 let args: Vec<String> = std::env::args().collect();
389 let mut i = 1;
390 while i < args.len() {
391 match args[i].as_str() {
392 "--save" if i + 1 < args.len() => {
393 i += 1;
394 save_path = Some(args[i].clone());
395 }
396 "--steps" if i + 1 < args.len() => {
397 i += 1;
398 train_steps = args[i].parse().unwrap_or(2000);
399 }
400 "--help" => {
401 usage(&args[0]);
402 return Ok(());
403 }
404 other if !positional_done && !other.starts_with('-') => {
405 data_dir = PathBuf::from(other);
406 positional_done = true;
407 }
408 other => {
409 eprintln!("unknown argument: {other}");
410 usage(&args[0]);
411 std::process::exit(1);
412 }
413 }
414 i += 1;
415 }
416
417 println!();
418 println!("╔══════════════════════════════════════════════════════════════════╗");
419 println!("║ OA Tutorial — Fashion-MNIST Prediction grid (oa::plot) ║");
420 println!("║ Train → Predict → display (5×5 grid, 25 test images) ║");
421 println!("║ Green title = correct · Red title = wrong ║");
422 println!(
423 "║ Mode: {:<58}║",
424 if save_path.is_some() {
425 "save_to (headless PNG output)"
426 } else {
427 "show (interactive window)"
428 }
429 );
430 println!("╚══════════════════════════════════════════════════════════════════╝");
431 println!();
432
433 // ── save mode: compute-only engine, no swapchain ─────────────────────────
434 if let Some(ref path) = save_path {
435 let engine = Engine::new()?;
436 Engine::set_default(engine.clone());
437 let cells = train_and_predict_grid(&engine, &data_dir, train_steps)?;
438 let mut fig = Figure::new(FigureConfig {
439 rows: GRID_ROWS as i32,
440 cols: GRID_COLS as i32,
441 width: 800,
442 height: 800,
443 h_spacing: 8,
444 v_spacing: 8,
445 padding: 4,
446 ..Default::default()
447 });
448 populate_figure(&mut fig, &cells);
449 fig.save_to(path)?;
450 println!("Saved: {path}");
451 return Ok(());
452 }
453
454 // ── show mode: Viewer lifecycle with a live figure source ─────────────────
455 let mut source = ImageGridClassifySource::new(data_dir, train_steps);
456 let config = oa::ViewerConfig {
457 mode: oa::ViewerMode::Live,
458 width: 800,
459 height: 800,
460 show_help: false,
461 show_stats: false,
462 show_timeline: false,
463 ..Default::default()
464 };
465 let mut viewer = oa::Viewer::new(config);
466 viewer.set_live_source(&mut source);
467
468 let window = oa::SdlWindow::new("OA — Fashion-MNIST grid", 800, 800)
469 .map_err(|e| anyhow::anyhow!("SDL window: {e}"))?;
470 let extensions = window
471 .required_instance_extensions()
472 .map_err(|e| anyhow::anyhow!("SDL extensions: {e}"))?;
473 let engine = Engine::builder().instance_extensions(extensions).build()?;
474 Engine::set_default(engine.clone());
475 viewer.run(&engine)?;
476 Ok(())
477}
478

Run Rust tutorials from the OA checkout with cargo run --release --example <name>. Run Python with the environment containing the OA native extension. Consult each program's arguments and capability checks before running it.