Ingestion Stage

from myspellchecker.data_pipeline import CorpusIngester

ingester = CorpusIngester()

# Generate sample corpus for testing
sample_lines = ingester.generate_sample_corpus()

# Process files into Arrow shards (used internally by Pipeline)
# The ingester produces Arrow shard files in the work directory

from myspellchecker.data_pipeline import Pipeline, PipelineConfig

config = PipelineConfig(
    text_col="text",         # Column name for CSV/TSV
    json_key="text",         # Key name for JSON
)

pipeline = Pipeline(config=config)
pipeline.build_database(
    input_files=["corpus.txt"],
    database_path="dict.db",
)

from myspellchecker.data_pipeline import PipelineConfig, IngesterConfig

# Pipeline-level configuration
pipeline_config = PipelineConfig(
    text_col="text",         # Column name for CSV/TSV
    json_key="text",         # Key name for JSON
    num_shards=20,           # Number of Arrow shards to create
)

# Ingester-specific configuration
ingester_config = IngesterConfig(
    batch_size=10000,        # Records per batch for Arrow writing
    encoding="utf-8",        # File encoding
    skip_empty_lines=True,   # Skip empty lines during ingestion
    normalize_unicode=True,  # Apply Unicode normalization
)

# Text files are read line by line
pipeline = Pipeline(config=PipelineConfig())
pipeline.build_database(
    input_files=["corpus.txt"],
    database_path="dict.db",
)

config = PipelineConfig(
    text_col="text",  # Column name containing text
)

config = PipelineConfig(
    json_key="text",  # Key name containing text
)

# Create a Parquet corpus file
import pyarrow as pa
import pyarrow.parquet as pq

table = pa.table({"text": ["မြန်မာစာ", "ကောင်းပါတယ်"]})
pq.write_table(table, "corpus.parquet")

# Then use with pipeline
pipeline.build_database(
    input_files=["corpus.parquet"],
    database_path="dict.db",
)

from myspellchecker.data_pipeline import Pipeline, PipelineConfig

# Large files are automatically handled with parallel sharding
config = PipelineConfig(
    num_shards=50,       # More shards for better parallelization
    batch_size=50000,    # Larger batches for throughput
)

pipeline = Pipeline(config=config)
pipeline.build_database(
    input_files=["large_corpus.txt"],
    database_path="dict.db",
)

from myspellchecker.data_pipeline import Pipeline, IngestionError

pipeline = Pipeline()

try:
    pipeline.build_database(
        input_files=["missing.txt"],
        database_path="dict.db",
    )
except IngestionError as e:
    print(f"Ingestion failed: {e}")
    print(f"Missing files: {e.missing_files}")

pipeline = Pipeline()
pipeline.build_database(
    input_files=[
        "corpus1.txt",
        "corpus2.txt",
        "data/corpus3.csv",
    ],
    database_path="dict.db",
)

myspellchecker build --input "corpus/*.txt" --output dict.db

# Optimized for large files
config = PipelineConfig(
    num_shards=50,      # More shards for parallelization
    num_workers=8,      # More workers
    batch_size=50000,   # Larger batches
)

from myspellchecker.data_pipeline import Pipeline, PipelineConfig

config = PipelineConfig(
    text_col="text",         # For CSV files
    json_key="text",         # For JSON files
)

pipeline = Pipeline(config=config)
pipeline.build_database(
    input_files=["corpus.txt"],
    database_path="dict.db",
)