Building Dictionaries

# Build from text corpus
myspellchecker build --input corpus.txt --output dict.db

# Build sample database for testing
myspellchecker build --sample

# Build with POS tagging
myspellchecker build --input corpus.txt --output dict.db --pos-tagger transformer

from myspellchecker.data_pipeline import Pipeline

pipeline = Pipeline()
pipeline.build_database(
    input_files=["corpus.txt"],
    database_path="dict.db",
)

from myspellchecker.data_pipeline import Pipeline, PipelineConfig

config = PipelineConfig(
    batch_size=10000,        # Records per batch
    num_shards=20,           # Shards for ingestion
    num_workers=4,           # Parallel workers (None = auto-detect)
    min_frequency=50,        # Minimum word frequency to include
    word_engine="myword",    # Word segmentation engine: "myword", "crf", or "transformer"
    keep_intermediate=False, # Keep intermediate Arrow files
    text_col="text",         # Column name for CSV/TSV
    json_key="text",         # Key name for JSON
)

pipeline = Pipeline(config=config)
pipeline.build_database(
    input_files=["corpus.txt"],
    database_path="dict.db",
)

pipeline.build_database(
    input_files=[
        "general_corpus.txt",
        "domain_specific.txt",
        "organization_names.txt",
    ],
    database_path="combined.db",
)

# CLI: rule-based (fast, no dependencies)
myspellchecker build --input corpus.txt --output dict.db --pos-tagger rule_based

# CLI: transformer (highest accuracy, requires GPU)
myspellchecker build --input corpus.txt --output dict.db \
  --pos-tagger transformer \
  --pos-model chuuhtetnaing/myanmar-pos-model \
  --pos-device 0

# Python API
from myspellchecker.core.config import POSTaggerConfig

config = PipelineConfig(
    pos_tagger=POSTaggerConfig(
        tagger_type="transformer",
        model_name="chuuhtetnaing/myanmar-pos-model",
        device=0,
    ),
)
pipeline = Pipeline(config=config)
pipeline.build_database(["corpus.txt"], "dict.db")

from myspellchecker.data_pipeline import DatabasePackager

packager = DatabasePackager.from_existing("dictionary.db")
stats = packager.apply_inferred_pos(
    min_frequency=0,
    min_confidence=0.0,
)
packager.close()
print(f"Inferred POS for {stats['inferred']} words")

word
ဆေးရုံ
ဆရာဝန်
လူနာ
ကုမ္ပဏီ

myspellchecker build --input corpus.txt --output dict.db \
  --curated-input curated_lexicon.csv

myspellchecker build --input new_data.txt --output existing.db --incremental

pipeline.build_database(
    input_files=["new_data.txt"],
    database_path="existing.db",
    incremental=True,
)

from myspellchecker.data_pipeline.config import PipelineConfig

config = PipelineConfig(
    # Master toggle
    enrich=True,

    # Individual toggles
    enrich_confusables=True,
    enrich_compounds=True,
    enrich_collocations=True,
    enrich_register=True,
)

PMI = log2( P(compound) / (P(w1) × P(w2)) )

-- Core tables
syllables(id, syllable, frequency)
words(id, word, syllable_count, frequency, pos_tag, is_curated,
      inferred_pos, inferred_confidence, inferred_source)
bigrams(id, word1_id, word2_id, probability, count)
trigrams(id, word1_id, word2_id, word3_id, probability, count)
fourgrams(id, word1_id, word2_id, word3_id, word4_id, probability, count)
fivegrams(id, word1_id, word2_id, word3_id, word4_id, word5_id, probability, count)

-- POS probability tables (for Viterbi tagger)
pos_unigrams(pos, probability)
pos_bigrams(pos1, pos2, probability)
pos_trigrams(pos1, pos2, pos3, probability)

-- Enrichment tables (from --no-enrich to skip)
confusable_pairs(id, word1, word2, confusion_type, context_overlap, freq_ratio, suppress, source)
compound_confusions(id, compound, part1, part2, compound_freq, split_freq, pmi)
collocations(id, word1, word2, pmi, npmi, count)
register_tags(word, register, confidence, formal_count, informal_count)

-- Metadata and file tracking
metadata(key, value)
processed_files(path, mtime, size)

# Lookup word frequency
cursor.execute("SELECT frequency FROM words WHERE word = ?", ("မြန်မာ",))

# Get bigram probability
cursor.execute("""
    SELECT b.probability
    FROM bigrams b
    JOIN words w1 ON b.word1_id = w1.id
    JOIN words w2 ON b.word2_id = w2.id
    WHERE w1.word = ? AND w2.word = ?
""", ("ထမင်း", "စား"))

from myspellchecker.data_pipeline import DatabasePackager

# Verify an existing database (from_existing() opens the connection)
packager = DatabasePackager.from_existing("output.db")
packager.verify_database()
packager.print_stats()
packager.close()

# Tune for large corpora
myspellchecker build --input huge_corpus.txt \
  --num-workers 8 \
  --batch-size 500000