Processing Stage

from myspellchecker.data_pipeline import CorpusSegmenter

segmenter = CorpusSegmenter(
    output_dir="intermediate/",
    word_engine="myword",  # "myword", "crf", or "transformer"
)

# Segment corpus from Arrow shards
segmented_path = segmenter.segment_corpus(input_files)

from myspellchecker.data_pipeline import Pipeline, PipelineConfig

config = PipelineConfig(
    word_engine="myword",  # Segmentation engine (default)
    num_workers=4,         # Parallel workers
)

pipeline = Pipeline(config=config)
pipeline.build_database(input_files, database_path)

from myspellchecker.data_pipeline import PipelineConfig, SegmenterConfig

# Via PipelineConfig (recommended)
config = PipelineConfig(
    word_engine="myword",    # "myword", "crf", or "transformer"
    num_workers=4,           # Parallel workers (None = auto)
    batch_size=10000,        # Records per batch
)

# Segmenter-specific configuration
segmenter_config = SegmenterConfig(
    batch_size=10000,
    word_engine="myword",
    num_workers=4,
    enable_pos_tagging=True,
    chunk_size=50000,        # Lines per chunk for parallel processing
)

config = PipelineConfig(
    word_engine="myword",
)

config = PipelineConfig(
    word_engine="crf",
)

config = PipelineConfig(
    word_engine="transformer",
    seg_model="chuuhtetnaing/myanmar-text-segmentation-model",  # Optional custom model
    seg_device=-1,  # -1=CPU, 0+=GPU
)

# Auto-detect CPU cores
config = PipelineConfig(num_workers=None)

# Manual setting
config = PipelineConfig(num_workers=8)

brew install libomp

# Small files
config = PipelineConfig(batch_size=1000)

# Large files
config = PipelineConfig(batch_size=50000)

config = PipelineConfig(
    batch_size=5000,  # Smaller batches
    num_workers=2,    # Fewer workers
)

from myspellchecker.data_pipeline import Pipeline, PipelineConfig

config = PipelineConfig(
    word_engine="myword",
    num_workers=4,
    batch_size=10000,
)

pipeline = Pipeline(config=config)
pipeline.build_database(
    input_files=["corpus.txt"],
    database_path="dict.db",
)