Tokenizers API

from myspellchecker.tokenizers import SyllableTokenizer

tokenizer = SyllableTokenizer()

tokenizer = SyllableTokenizer()

# Basic syllable tokenization
syllables = tokenizer.tokenize("မြန်မာနိုင်ငံ")
# Output: ['မြန်', 'မာ', 'နိုင်', 'ငံ']

# Handles stacked consonants correctly
syllables = tokenizer.tokenize("သင်္ဘော")
# Output: ['သင်္ဘော']  # Kinzi preserved as single unit

# Handles mixed content
syllables = tokenizer.tokenize("မြန်မာ123abc")
# Output: ['မြန်', 'မာ', '1', '2', '3', 'a', 'b', 'c']

# Internal pattern logic (simplified)
pattern = r"((?<!္)[က-အ](?![်္])|[a-zA-Z0-9\s...])"

from myspellchecker.tokenizers import WordTokenizer

# Default: myword engine (recommended)
tokenizer = WordTokenizer(engine="myword")

# Alternative: CRF engine
tokenizer = WordTokenizer(engine="CRF")

tokenizer = WordTokenizer(engine="myword")

# Word segmentation
words = tokenizer.tokenize("မြန်မာနိုင်ငံသည်အရှေ့တောင်အာရှတွင်တည်ရှိသည်")
# Output: ['မြန်မာ', 'နိုင်ငံ', 'သည်', 'အရှေ့တောင်', 'အာရှ', 'တွင်', 'တည်ရှိ', 'သည်']

# Handles numerals
words = tokenizer.tokenize("လူ၃ယောက်")
# Output: ['လူ', '၃', 'ယောက်']

1. Load segmentation.mmap file
2. Initialize Cython mmap reader
3. Configure Viterbi function

tokenizer = WordTokenizer(engine="myword")

# Check if custom words exist in the mmap dictionary
tokenizer.add_custom_words(["ဆော့ဖ်ဝဲ", "ဒေတာဘေ့စ်"])
# Logs: "2/2 words found in dictionary." or warnings for missing words

# Automatic normalization
words = tokenizer.tokenize("ဝါကျ")  # wa as letter
# Output: ['ဝါကျ']

words = tokenizer.tokenize("၂၀၂၄")  # zeros in number preserved
# Output: ['၂၀၂၄']

tokenizer = WordTokenizer(engine="myword")

# Check if using Cython
print(f"Using Cython: {tokenizer._using_cython}")
print(f"Using mmap: {tokenizer._using_mmap}")

from myspellchecker.tokenizers import WordTokenizer

# Invalid engine
try:
    tokenizer = WordTokenizer(engine="invalid")
except ValueError as e:
    print(e)  # "Unknown engine: invalid. Must be one of: CRF, myword"

# Missing mmap file
try:
    tokenizer = WordTokenizer(engine="myword")
except RuntimeError as e:
    print(e)  # "segmentation.mmap is required for myword engine..."

pip install myspellchecker[transformers]

from myspellchecker.tokenizers.transformer_word_segmenter import (
    TransformerWordSegmenter,
)

# Use the default model
segmenter = TransformerWordSegmenter()

# Use a custom model
segmenter = TransformerWordSegmenter(
    model_name="your-org/your-model",
    device=0,  # GPU
)

segmenter = TransformerWordSegmenter()

# Single text segmentation
words = segmenter.segment("မြန်မာနိုင်ငံသည်")
# Output depends on model: e.g., ['မြန်မာ', 'နိုင်ငံ', 'သည်']

# Batch segmentation (more efficient for multiple texts)
results = segmenter.segment_batch([
    "မြန်မာနိုင်ငံ",
    "ကျွန်တော်သွားပါမယ်",
])
# Output: list of word lists, one per input text

Input tokens:  [B:"မြန်", I:"မာ", B:"နိုင်", I:"ငံ", B:"သည်"]
Merged words:  ["မြန်မာ",          "နိုင်ငံ",          "သည်"]

# Efficient: single batch call
results = segmenter.segment_batch(texts)

# Inefficient: individual calls
results = [segmenter.segment(t) for t in texts]

# Build with transformer word segmentation
myspellchecker build \
    --input corpus.txt \
    --output dict.db \
    --word-engine transformer

# Use a custom model
myspellchecker build \
    --input corpus.txt \
    --output dict.db \
    --word-engine transformer \
    --seg-model "your-org/your-model"

# Use GPU (CUDA or MPS auto-detected)
myspellchecker build \
    --input corpus.txt \
    --output dict.db \
    --word-engine transformer \
    --seg-device 0

from myspellchecker.data_pipeline import Pipeline, PipelineConfig

config = PipelineConfig(
    word_engine="transformer",
    seg_model="your-org/your-model",  # optional, uses default if None
    seg_device=0,                     # optional, -1 for CPU
)

pipeline = Pipeline(config=config)
pipeline.build_database(
    input_files=["corpus.txt"],
    database_path="dict.db",
)

# Missing transformers package
try:
    segmenter = TransformerWordSegmenter()
except ImportError as e:
    print(e)
    # "Transformer-based word segmentation requires the 'transformers' library.
    #  Install with: pip install myspellchecker[transformers]"

# Invalid model
try:
    segmenter = TransformerWordSegmenter(model_name="nonexistent/model")
except ValueError as e:
    print(e)
    # "Failed to load model 'nonexistent/model': ..."