oa::ml::tokenizer::BpeMerge

One learned byte-pair merge. Merge rank i creates token 256 + i.

StRust public API

Declaration

pub struct BpeMerge { pub left: u32, pub right: u32, }

Public Rust declaration.

fn clone(&self) -> BpeMerge

fn fmt(&self, f: &mut Formatter<'_>) -> Result

fn eq(&self, other: &BpeMerge) -> bool

leftu32field

rightu32field

fn clone(
	&self
) -> BpeMerge

No public source comment is attached to this declaration.

fn fmt(
	&self,
	f: &mut Formatter<'_>
) -> Result

No public source comment is attached to this declaration.

fn eq(
	&self,
	other: &BpeMerge
) -> bool

No public source comment is attached to this declaration.