"""Scan extracted text for OCR defects and generate a fix-config template.

Scanned PDFs mangle text in systematic ways:
  - U+FFFD (replacement character) where the OCR engine gave up, usually on
    diacritics and unusual consonants (foreign-language terms, umlauts, ...)
  - Confused characters: ; -> s/h, J -> th/v, 9 -> d, ! -> l, < -> sh, ...

The OCR engine of a given scan misreads the SAME glyph the SAME way, so the
right tool is an exact-string replacement map built from the actual output.
`scan` enumerates the distinct suspicious tokens with counts and surrounding
context; you (or a language model) then fill in the correct reading, and
`clean` applies the map.

Output of scan:
  - a human-readable report on stdout
  - optionally a fix-config template JSON with the suspicious tokens as
    empty values, ready to be filled in:  {"fixes": {"mangled-token": ""}}
"""
import json
import re
import sys

FFFD = "\ufffd"

# Characters that rarely appear inside a real English word. A token that has
# one of these in the MIDDLE of the word (or FFFD anywhere) is suspect.
SUSPECT = set(";<>![]\\_`|~^@#%&*+=")

TOKEN_RE = re.compile(r"[\w" + FFFD + r";<>!\\[\]_`\-]+")


def is_suspect(token: str) -> bool:
    if FFFD in token:
        return True
    body = token.strip("-_")
    if len(body) < 3:
        return False
    # suspect char inside the word (not a leading/trailing punctuation)
    return any(c in SUSPECT for c in body[1:-1])


def scan(text: str, top: int = 300) -> list:
    """Return [(count, token, context)] sorted by count desc, then token."""
    counts = {}
    context = {}
    for m in TOKEN_RE.finditer(text):
        tok = m.group(0)
        if not is_suspect(tok):
            continue
        counts[tok] = counts.get(tok, 0) + 1
        if tok not in context:
            s = max(0, m.start() - 40)
            e = min(len(text), m.end() + 40)
            ctx = text[s:e].replace("\n", " ")
            context[tok] = ctx
    items = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
    return [(c, t, context[t]) for t, c in items[:top]]


def cmd_scan(path: str, top: int = 300, template: str = None) -> None:
    text = open(path, encoding="utf-8").read()
    total_fffd = text.count(FFFD)
    words = len(text.split())
    items = scan(text, top=top)

    print(f"text: {path}")
    print(f"words: {words}  U+FFFD glyphs: {total_fffd}  "
          f"suspect tokens (distinct): {len(items)}")
    print()
    print(f"{'count':>6}  {'token':<32}  context")
    print("-" * 100)
    for count, tok, ctx in items:
        print(f"{count:>6}  {tok!r:<32}  ...{ctx}...")

    if template:
        fixes = {tok: "" for _, tok, _ in items}
        cfg = {
            "include": {"start": None, "end": None},
            "drop_blocks": [],
            "page_header_patterns": [],
            "drop_line_patterns": [],
            "strip_patterns": [],
            "raw_fixes": {},
            "regex_fixes": [],
            "word_fixes": [],
            "fixes": fixes,
            "heading_pattern": None,
            "heading_count_max": 150,
        }
        with open(template, "w", encoding="utf-8") as f:
            json.dump(cfg, f, indent=2, ensure_ascii=False)
        print(f"\nfix-config template written to {template} "
              f"({len(fixes)} tokens to review). Fill in the empty values "
              f"with the correct reading; empty values are ignored by clean.")
        print("See README 'Fixing OCR defects' for the workflow.")


if __name__ == "__main__":
    cmd_scan(sys.argv[1])