RealmOA Library

oa::ml::tokenizer::BpeTokenizer

Deterministic byte-pair encoder compatible with OA’s oa_bpe_v1 files.

StRust public API

Declaration

pub struct BpeTokenizer { /* private fields */ }
Public Rust declaration.

Public Methods

pub fn train(&mut self, bytes: &[u8], num_merges: usize)
pub fn train_text(&mut self, text: &str, num_merges: usize)
pub fn encode(&self, bytes: &[u8]) -> Vec<u32>
pub fn encode_text(&self, text: &str) -> Vec<u32>
pub fn decode(&self, tokens: &[u32]) -> Result<Vec<u8>>
pub fn decode_text(&self, tokens: &[u32]) -> Result<String>
pub fn encode_prompt(&self, prompt: &[u8], context_length: usize) -> Vec<u32>
pub fn save(&self, path: impl AsRef<Path>) -> Result<()>
pub fn load(&mut self, path: impl AsRef<Path>) -> Result<()>
pub const fn vocab_size(&self) -> usize
pub const fn num_merges(&self) -> usize
pub fn merges(&self) -> &[BpeMerge]
pub fn token_byte_len(&self, token: u32) -> Result<usize>

Associated Functions

pub fn new(target_vocab: usize) -> Self
fn default() -> Self

Public Method Documentation

pub fn train( &mut self, bytes: &[u8], num_merges: usize )
No public source comment is attached to this declaration.
pub fn train_text( &mut self, text: &str, num_merges: usize )
No public source comment is attached to this declaration.
pub fn encode( &self, bytes: &[u8] ) -> Vec<u32>
No public source comment is attached to this declaration.
pub fn encode_text( &self, text: &str ) -> Vec<u32>
No public source comment is attached to this declaration.
pub fn decode( &self, tokens: &[u32] ) -> Result<Vec<u8>>
No public source comment is attached to this declaration.
pub fn decode_text( &self, tokens: &[u32] ) -> Result<String>
No public source comment is attached to this declaration.
pub fn encode_prompt( &self, prompt: &[u8], context_length: usize ) -> Vec<u32>
No public source comment is attached to this declaration.
pub fn save(&self, path: impl AsRef<Path>) -> Result<()>
No public source comment is attached to this declaration.
pub fn load(&mut self, path: impl AsRef<Path>) -> Result<()>
No public source comment is attached to this declaration.
pub const fn vocab_size( &self ) -> usize
No public source comment is attached to this declaration.
pub const fn num_merges( &self ) -> usize
No public source comment is attached to this declaration.
pub fn merges( &self ) -> &[BpeMerge]
No public source comment is attached to this declaration.
pub fn token_byte_len( &self, token: u32 ) -> Result<usize>
No public source comment is attached to this declaration.
pub fn new( target_vocab: usize ) -> Self
No public source comment is attached to this declaration.
fn default() -> Self
No public source comment is attached to this declaration.