Segmentation Repair

from myspellchecker.data_pipeline.repair import SegmentationRepair

repairer = SegmentationRepair()

# Fix broken segmentation
tokens = ["ကျော", "င်း", "သည်"]
repaired = repairer.repair(tokens)
print(repaired)  # ["ကျောင်း", "သည်"]

Input text: "ကျောင်းသည်"
Bad segmentation: ["ကျော", "င်း", "သည်"]
                    ↑       ↑
                    Broken syllable "ကျောင်း"

Good segmentation: ["ကျောင်း", "သည်"]

class SegmentationRepair:
    """Repairs invalid word segmentations by merging broken syllables."""

    def __init__(self):
        self.validator = SyllableRuleValidator(allow_extended_myanmar=allow_extended_myanmar)

        # Pattern for valid syllable starters
        self.valid_start_pattern = re.compile(
            rf"^[{''.join(CONSONANTS)}{''.join(INDEPENDENT_VOWELS)}{''.join(MYANMAR_NUMERALS)}]"
        )

        # Pattern for Myanmar numerals (skip validation)
        self.numeral_start_pattern = re.compile(
            rf"^[{''.join(MYANMAR_NUMERALS)}]"
        )

        # Known problematic fragments
        self.suspicious_fragments = {
            "င်း",  # Part of ောင်း, ိုင်း
            "င့်",  # Part of ောင့်
            "န့်",  # Part of ောင့်
        }

        # Characters indicating closed syllable
        self.closing_chars = {"း", "့", "်"}

        # Fragments causing double-ending
        self.double_ending_fragments = {"င်း", "င့်", "န့်"}

def repair(self, tokens: List[str]) -> List[str]:
    """Repair a list of tokens by merging invalid fragments."""
    if not tokens:
        return []

    repaired = [tokens[0]]

    for i in range(1, len(tokens)):
        current_token = tokens[i]
        prev_token = repaired[-1]

        # Check if current token is a fragment
        is_invalid_start = not self.valid_start_pattern.match(current_token)
        is_suspicious = current_token in self.suspicious_fragments
        is_numeral = self.numeral_start_pattern.match(current_token)

        # Skip syllable validation for numerals
        is_invalid_syllable = False
        if not is_invalid_start and not is_suspicious and not is_numeral:
            is_invalid_syllable = not self.validator.validate(current_token)

        if is_invalid_start or is_suspicious or is_invalid_syllable:
            # Attempt merge with previous token
            # ... (merge logic)
        else:
            # Valid token, keep separate
            repaired.append(current_token)

    return repaired

# Inline merge logic in repair() (simplified):
if self._would_create_double_ending(prev_token, current_token):
    repaired.append(current_token)  # Reject merge
    continue

prev_is_closed = prev_token and prev_token[-1] in self.closing_chars
if prev_is_closed:
    repaired.append(current_token)  # Reject merge
    continue

candidate = prev_token + current_token
if self.validator.validate(candidate):
    repaired[-1] = candidate        # Accept merge
else:
    repaired.append(current_token)  # Reject merge

def _would_create_double_ending(self, prev_token: str, current_token: str) -> bool:
    """Check if merging would create invalid double-ending."""
    if not prev_token or not current_token:
        return False

    # Closed syllable + double-ending fragment = invalid
    if prev_token[-1] in self.closing_chars:
        if current_token in self.double_ending_fragments:
            return True

    return False

repairer = SegmentationRepair()

# Fix broken syllable
tokens = ["ကျော", "င်း"]
result = repairer.repair(tokens)
# Result: ["ကျောင်း"]

# Multiple fragments
tokens = ["မြန်", "မာ", "နိုင်", "င", "ံ"]
result = repairer.repair(tokens)
# Result: ["မြန်", "မာ", "နိုင်ငံ"]

# Valid tokens are preserved
tokens = ["ကျောင်း", "သည်", "ကောင်းသည်"]
result = repairer.repair(tokens)
# Result: ["ကျောင်း", "သည်", "ကောင်းသည်"] (unchanged)

# Numerals are not merged
tokens = ["၁", "၂", "၃"]
result = repairer.repair(tokens)
# Result: ["၁", "၂", "၃"] (unchanged)

# Mixed text and numerals
tokens = ["အမှတ်", "၁", "၀", "၀"]
result = repairer.repair(tokens)
# Result: ["အမှတ်", "၁", "၀", "၀"]

# Don't merge with closed syllables
tokens = ["ပြီး", "င်း"]
result = repairer.repair(tokens)
# Result: ["ပြီး", "င်း"] (not merged - "ပြီး" is closed)

# Valid merge with open syllable
tokens = ["ကျော", "င်း"]
result = repairer.repair(tokens)
# Result: ["ကျောင်း"] (merged - "ကျော" is open)

from myspellchecker.data_pipeline.repair import SegmentationRepair

def process_corpus(lines: List[str]) -> List[List[str]]:
    """Process corpus with segmentation repair."""
    from myspellchecker.segmenters import DefaultSegmenter
    segmenter = DefaultSegmenter()
    repairer = SegmentationRepair()

    results = []
    for line in lines:
        words = segmenter.segment_words(line)
        repaired = repairer.repair(words)
        results.append(repaired)

    return results

# Repair is used internally during pipeline builds:
from myspellchecker.data_pipeline.segmenter import CorpusSegmenter

segmenter = CorpusSegmenter(word_engine="myword")
# CorpusSegmenter applies repair logic internally during segmentation

try:
    from myspellchecker.data_pipeline.repair_c import CythonSegmentationRepair as SegmentationRepair
except ImportError:
    from myspellchecker.data_pipeline.repair import SegmentationRepair

# Same API, faster execution
repairer = SegmentationRepair()
repaired = repairer.repair(tokens)