bitscuit
a1796e82aa
Pipeline: extract (pdftotext) -> scan (OCR-defect report + fix template) -> clean (noise/header removal, hyphenation, exact-string/regex OCR repairs, chapter-heading detection) -> voices (sample + comparison) -> chunk (sentence-aware) -> synth (resumable edge-tts) -> stitch (ffmpeg concat + RMS verify) -> chapters (ID3v2.4 CHAP). Config-driven per book; validated end-to-end on a clean digital PDF and on a 546-page scanned book (byte-identical clean output to the reference run, 95 auto-detected chapters).
100 lines
3.5 KiB
Python
100 lines
3.5 KiB
Python
"""Scan extracted text for OCR defects and generate a fix-config template.
|
|
|
|
Scanned PDFs mangle text in systematic ways:
|
|
- U+FFFD (replacement character) where the OCR engine gave up, usually on
|
|
diacritics and unusual consonants (foreign-language terms, umlauts, ...)
|
|
- Confused characters: ; -> s/h, J -> th/v, 9 -> d, ! -> l, < -> sh, ...
|
|
|
|
The OCR engine of a given scan misreads the SAME glyph the SAME way, so the
|
|
right tool is an exact-string replacement map built from the actual output.
|
|
`scan` enumerates the distinct suspicious tokens with counts and surrounding
|
|
context; you (or a language model) then fill in the correct reading, and
|
|
`clean` applies the map.
|
|
|
|
Output of scan:
|
|
- a human-readable report on stdout
|
|
- optionally a fix-config template JSON with the suspicious tokens as
|
|
empty values, ready to be filled in: {"fixes": {"mangled-token": ""}}
|
|
"""
|
|
import json
|
|
import re
|
|
import sys
|
|
|
|
FFFD = "\ufffd"
|
|
|
|
# Characters that rarely appear inside a real English word. A token that has
|
|
# one of these in the MIDDLE of the word (or FFFD anywhere) is suspect.
|
|
SUSPECT = set(";<>![]\\_`|~^@#%&*+=")
|
|
|
|
TOKEN_RE = re.compile(r"[\w" + FFFD + r";<>!\\[\]_`\-]+")
|
|
|
|
|
|
def is_suspect(token: str) -> bool:
|
|
if FFFD in token:
|
|
return True
|
|
body = token.strip("-_")
|
|
if len(body) < 3:
|
|
return False
|
|
# suspect char inside the word (not a leading/trailing punctuation)
|
|
return any(c in SUSPECT for c in body[1:-1])
|
|
|
|
|
|
def scan(text: str, top: int = 300) -> list:
|
|
"""Return [(count, token, context)] sorted by count desc, then token."""
|
|
counts = {}
|
|
context = {}
|
|
for m in TOKEN_RE.finditer(text):
|
|
tok = m.group(0)
|
|
if not is_suspect(tok):
|
|
continue
|
|
counts[tok] = counts.get(tok, 0) + 1
|
|
if tok not in context:
|
|
s = max(0, m.start() - 40)
|
|
e = min(len(text), m.end() + 40)
|
|
ctx = text[s:e].replace("\n", " ")
|
|
context[tok] = ctx
|
|
items = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
return [(c, t, context[t]) for t, c in items[:top]]
|
|
|
|
|
|
def cmd_scan(path: str, top: int = 300, template: str = None) -> None:
|
|
text = open(path, encoding="utf-8").read()
|
|
total_fffd = text.count(FFFD)
|
|
words = len(text.split())
|
|
items = scan(text, top=top)
|
|
|
|
print(f"text: {path}")
|
|
print(f"words: {words} U+FFFD glyphs: {total_fffd} "
|
|
f"suspect tokens (distinct): {len(items)}")
|
|
print()
|
|
print(f"{'count':>6} {'token':<32} context")
|
|
print("-" * 100)
|
|
for count, tok, ctx in items:
|
|
print(f"{count:>6} {tok!r:<32} ...{ctx}...")
|
|
|
|
if template:
|
|
fixes = {tok: "" for _, tok, _ in items}
|
|
cfg = {
|
|
"include": {"start": None, "end": None},
|
|
"drop_blocks": [],
|
|
"page_header_patterns": [],
|
|
"drop_line_patterns": [],
|
|
"strip_patterns": [],
|
|
"raw_fixes": {},
|
|
"regex_fixes": [],
|
|
"word_fixes": [],
|
|
"fixes": fixes,
|
|
"heading_pattern": None,
|
|
"heading_count_max": 150,
|
|
}
|
|
with open(template, "w", encoding="utf-8") as f:
|
|
json.dump(cfg, f, indent=2, ensure_ascii=False)
|
|
print(f"\nfix-config template written to {template} "
|
|
f"({len(fixes)} tokens to review). Fill in the empty values "
|
|
f"with the correct reading; empty values are ignored by clean.")
|
|
print("See README 'Fixing OCR defects' for the workflow.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cmd_scan(sys.argv[1])
|