Text Normalization

from myspellchecker.text.normalization_service import (
    NormalizationService,
    get_normalization_service
)

# Get singleton service
service = get_normalization_service()

# Or create new instance
service = NormalizationService()

normalized = service.for_spell_checking("  မြန်မာ  ")
print(normalized)  # "မြန်မာ"

normalized = service.for_dictionary_lookup(user_input)

a = service.for_comparison(user_input)
b = service.for_comparison(dictionary_entry)
if a == b:
    print("Match!")

normalized = service.for_display(text)

normalized = service.for_ingestion(corpus_line)

from myspellchecker.text.normalization_service import (
    NormalizationService,
    NormalizationOptions
)

options = NormalizationOptions(
    unicode_form="NFC",         # NFC, NFD, NFKC, NFKD
    remove_zero_width=True,     # Remove zero-width characters
    reorder_diacritics=True,    # Myanmar-specific reordering
    convert_zawgyi=False,       # Detect and convert Zawgyi
    strip_whitespace=True,      # Strip leading/trailing whitespace
    lowercase=False,            # Lowercase (for non-Myanmar text)
    character_variants=False    # Normalize Myanmar character variants (Tall-AA, etc.)
)

service = NormalizationService()
normalized = service.normalize(text, options)

from myspellchecker.text.normalization_service import (
    PRESET_SPELL_CHECK,
    PRESET_DICTIONARY_LOOKUP,
    PRESET_COMPARISON,
    PRESET_DISPLAY,
    PRESET_INGESTION
)

service = NormalizationService()
normalized = service.normalize(text, PRESET_COMPARISON)

service = NormalizationService()

is_myanmar = service.is_myanmar_text("မြန်မာ")  # True
is_myanmar = service.is_myanmar_text("Hello")  # False
is_myanmar = service.is_myanmar_text("Hello မြန်မာ")  # Depends on threshold

# Include Extended Myanmar blocks (Shan, Mon, etc.)
is_myanmar = service.is_myanmar_text(text, allow_extended=True)

from myspellchecker.core.config.text_configs import ZawgyiConfig

# Custom Zawgyi configuration
zawgyi_config = ZawgyiConfig(
    conversion_threshold=0.9,     # Probability threshold for conversion
    myanmar_text_threshold=0.3    # Min Myanmar character ratio
)

service = NormalizationService(zawgyi_config=zawgyi_config)

# Will convert Zawgyi if probability >= 0.9
normalized = service.for_dictionary_lookup(potentially_zawgyi_text)

from myspellchecker.text.normalization_service import (
    normalize_for_spell_checking,
    normalize_for_lookup,
    normalize_for_comparison
)

# These use the default singleton service
normalized = normalize_for_spell_checking(text)
normalized = normalize_for_lookup(text)
normalized = normalize_for_comparison(text)

# These are used internally by NormalizationService
from myspellchecker.text.normalize_c import (
    remove_zero_width_chars,      # Fast zero-width removal
    reorder_myanmar_diacritics,   # Diacritic reordering
    get_myanmar_ratio             # Myanmar character ratio
)

from concurrent.futures import ThreadPoolExecutor

service = get_normalization_service()  # Thread-safe singleton

with ThreadPoolExecutor(max_workers=4) as executor:
    futures = [
        executor.submit(service.for_spell_checking, text)
        for text in texts
    ]
    results = [f.result() for f in futures]

from myspellchecker import SpellChecker
from myspellchecker.providers import SQLiteProvider

provider = SQLiteProvider(database_path="path/to/dictionary.db")
checker = SpellChecker(provider=provider)
# Internally uses NormalizationService for consistent normalization
result = checker.check(text)

from myspellchecker.data_pipeline import Pipeline

pipeline = Pipeline()
# Uses for_ingestion() when processing corpus files
pipeline.build_database(input_files=["corpus.txt"], database_path="output.db")

from myspellchecker.providers import SQLiteProvider

provider = SQLiteProvider()
# Uses normalized form before database queries
is_valid = provider.is_valid_word("မြန်မာ")

import unicodedata

# Composed form (NFC)
text = unicodedata.normalize("NFC", text)

# Example: the syllable ကော ("kaw")
# Before: U+1031 U+1000 U+102C  (ေ stored before က — non-canonical)
# After:  U+1000 U+1031 U+102C  (က before ေ — canonical Unicode order)
# Both render visually as: ကော
# The ေ vowel always appears to the left visually, regardless of codepoint order.

# Requires myanmar-tools package
# pip install myanmar-tools