ocr-postprocess 0.1.0__py3-none-any.whl

ocr_postprocess/__init__.py

@@ -0,0 +1,33 @@
+"""ocr_postprocess — OCR post-processing pipeline."""
+
+__version__ = "0.1.0"
+
+from ocr_postprocess.exceptions import (
+    CyclicComputeError,
+    ExtractorNotFoundError,
+    OcrPostprocessError,
+    ProfileNotFoundError,
+    ProfileValidationError,
+    TransformError,
+)
+from ocr_postprocess.models import PipelineContext, ProcessedDocument
+from ocr_postprocess.pipeline import Pipeline
+from ocr_postprocess.renderer.llm import render_llm_markdown
+
+__all__ = [
+    # Core
+    "Pipeline",
+    "ProcessedDocument",
+    "PipelineContext",
+    # Renderers
+    "render_llm_markdown",
+    # Exceptions — import these to catch errors without knowing internal paths
+    "OcrPostprocessError",
+    "ProfileNotFoundError",
+    "ProfileValidationError",
+    "ExtractorNotFoundError",
+    "TransformError",
+    "CyclicComputeError",
+    # Version
+    "__version__",
+]

ocr_postprocess/classifier.py

@@ -0,0 +1,63 @@
+"""Stage 2 — Classifier: select best matching DocumentProfile."""
+
+from __future__ import annotations
+
+import logging
+
+from ocr_postprocess.models import PipelineContext
+from ocr_postprocess.profiles.matcher import evaluate
+
+logger = logging.getLogger(__name__)
+
+
+def classify_stage(ctx: PipelineContext) -> None:
+    """Pipeline stage 2: classify document and load matching profile."""
+    profiles: dict = ctx.__dict__.get("_profiles", {})
+
+    if not profiles:
+        logger.warning("No profiles loaded; using generic fallback")
+        ctx.classification_score = 0.0
+        return
+
+    text = ctx.normalized_text or ctx.raw_text
+    best_id: str | None = None
+    best_score = 0.0
+    second_score = 0.0
+
+    scores: list[tuple[float, str]] = []
+    for pid, profile in profiles.items():
+        if pid.startswith("_"):
+            continue  # skip generic fallback in main scoring
+        try:
+            score = evaluate(profile.classify, text)
+        except Exception:
+            logger.exception("Error evaluating classify for profile '%s'", pid)
+            score = 0.0
+        scores.append((score, pid))
+
+    scores.sort(reverse=True)
+
+    if scores:
+        best_score, best_id = scores[0]
+        second_score = scores[1][0] if len(scores) > 1 else 0.0
+
+        if best_score < 0.5:
+            # Fall back to _generic
+            best_id = "_generic"
+            best_score = 0.0
+            logger.info("Classification score too low; using _generic")
+        elif len(scores) > 1 and best_score - second_score < 0.1:
+            logger.warning(
+                "Ambiguous classification: '%s'=%.2f vs '%s'=%.2f",
+                best_id,
+                best_score,
+                scores[1][1],
+                second_score,
+            )
+    else:
+        best_id = "_generic"
+
+    profile_obj = profiles.get(best_id)
+    ctx.profile = profile_obj
+    ctx.classification_score = best_score
+    logger.info("Classified as '%s' (score=%.3f)", best_id, best_score)

ocr_postprocess/cli.py

@@ -0,0 +1,130 @@
+"""Typer CLI for ocr_postprocess."""
+
+from __future__ import annotations
+
+import logging
+import sys
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+app = typer.Typer(name="ocrpp", help="OCR post-processing pipeline CLI.", add_completion=False)
+
+_LOG_LEVELS = {
+    "debug": logging.DEBUG,
+    "info": logging.INFO,
+    "warning": logging.WARNING,
+    "error": logging.ERROR,
+}
+
+
+def _setup_logging(level: str) -> None:
+    """Configure root logger with given level string."""
+    logging.basicConfig(
+        level=_LOG_LEVELS.get(level.lower(), logging.INFO),
+        format="%(levelname)s %(name)s: %(message)s",
+    )
+
+
+@app.command()
+def classify(
+    input_file: Optional[Path] = typer.Argument(
+        None, help="Path to raw text file. Reads stdin if omitted."
+    ),
+    profiles_dir: Path = typer.Option(
+        Path("profiles"), "--profiles", "-p", help="Profiles directory."
+    ),
+    log_level: str = typer.Option("info", "--log-level", "-l"),
+) -> None:
+    """Classify a document and print the best matching profile ID."""
+    _setup_logging(log_level)
+    from ocr_postprocess.pipeline import Pipeline
+
+    raw = _read_input(input_file)
+    pipeline = Pipeline.from_default(profiles_dir=str(profiles_dir))
+    profile_id, score = pipeline.classify(raw)
+    typer.echo(f"{profile_id}\t{score:.4f}")
+
+
+@app.command()
+def process(
+    input_file: Optional[Path] = typer.Argument(
+        None, help="Path to raw text file. Reads stdin if omitted."
+    ),
+    profiles_dir: Path = typer.Option(
+        Path("profiles"), "--profiles", "-p", help="Profiles directory."
+    ),
+    output_format: str = typer.Option(
+        "json", "--format", "-f", help="Output format: json, markdown, or llm."
+    ),
+    debug: bool = typer.Option(False, "--debug", "-d", help="Include debug trace."),
+    log_level: str = typer.Option("info", "--log-level", "-l"),
+) -> None:
+    """Process a document and print extracted fields."""
+    _setup_logging(log_level)
+    from ocr_postprocess.pipeline import Pipeline
+    from ocr_postprocess.renderer.json_renderer import to_json
+    from ocr_postprocess.renderer.llm import render_llm_markdown
+    from ocr_postprocess.renderer.markdown import render_markdown
+
+    raw = _read_input(input_file)
+    pipeline = Pipeline.from_default(profiles_dir=str(profiles_dir))
+    doc = pipeline.process(raw, debug=debug)
+
+    if output_format == "markdown":
+        typer.echo(render_markdown(doc))
+    elif output_format == "llm":
+        typer.echo(render_llm_markdown(doc))
+    else:
+        typer.echo(to_json(doc))
+
+
+@app.command("validate-profile")
+def validate_profile(
+    profile_file: Path = typer.Argument(..., help="Path to YAML profile file."),
+    log_level: str = typer.Option("info", "--log-level", "-l"),
+) -> None:
+    """Validate a YAML profile file and print errors if any."""
+    _setup_logging(log_level)
+    from ocr_postprocess.profiles.loader import load_profile
+
+    try:
+        profile = load_profile(profile_file)
+        typer.echo(f"OK: {profile.id} (v{profile.version})")
+    except Exception as exc:
+        typer.echo(f"ERROR: {exc}", err=True)
+        raise typer.Exit(code=1)
+
+
+@app.command("dump-canonical")
+def dump_canonical(
+    input_file: Optional[Path] = typer.Argument(None, help="Path to raw text file."),
+    profiles_dir: Path = typer.Option(
+        Path("profiles"), "--profiles", "-p", help="Profiles directory."
+    ),
+    log_level: str = typer.Option("info", "--log-level", "-l"),
+) -> None:
+    """Process document and dump canonical JSON to stdout."""
+    _setup_logging(log_level)
+    from ocr_postprocess.pipeline import Pipeline
+    from ocr_postprocess.renderer.json_renderer import to_json
+
+    raw = _read_input(input_file)
+    pipeline = Pipeline.from_default(profiles_dir=str(profiles_dir))
+    doc = pipeline.process(raw)
+    typer.echo(to_json(doc))
+
+
+def _read_input(path: Optional[Path]) -> str:
+    """Read text from file path or stdin; exit with code 1 if no input available."""
+    if path is not None:
+        return path.read_text(encoding="utf-8")
+    if not sys.stdin.isatty():
+        return sys.stdin.read()
+    typer.echo("Error: no input provided", err=True)
+    raise typer.Exit(code=1)
+
+
+if __name__ == "__main__":
+    app()

ocr_postprocess/engine/denoiser.py

@@ -0,0 +1,134 @@
+"""Stage 3 — Denoiser: remove boilerplate lines and patterns."""
+
+from __future__ import annotations
+
+import logging
+
+import regex as re
+
+from ocr_postprocess.models import PipelineContext
+
+logger = logging.getLogger(__name__)
+
+# Built-in rules (always applied regardless of profile config)
+_BUILTIN_DROP_PATTERNS = [
+    re.compile(
+        r"^[\W_]{3,}$"
+    ),  # lines consisting only of non-alphanumeric chars (e.g. "---", "***")
+    re.compile(r"^\s{0,2}\S\s{0,2}$"),  # single character with optional surrounding spaces
+]
+
+
+def _is_builtin_noise(line: str) -> bool:
+    """Return True if line matches built-in noise rules."""
+    stripped = line.strip()
+    if len(stripped) < 3:
+        return True
+    for pat in _BUILTIN_DROP_PATTERNS:
+        if pat.fullmatch(stripped):
+            return True
+    return False
+
+
+def denoise(
+    text: str,
+    drop_line_patterns: list[str] | None = None,
+    drop_inline_patterns: list[str] | None = None,
+    mask_patterns: list[dict[str, str]] | None = None,
+    collapse_repeats: bool = False,
+    protected_substrings: set[str] | None = None,
+) -> str:
+    """Remove noise from text, preserving line count (empty string for dropped lines)."""
+    compiled_drop = [re.compile(p) for p in (drop_line_patterns or [])]
+    compiled_inline = [re.compile(p) for p in (drop_inline_patterns or [])]
+    compiled_mask = [
+        (re.compile(m["pattern"]), m.get("replacement", "")) for m in (mask_patterns or [])
+    ]
+
+    lines = text.split("\n")
+    prev_line: str | None = None
+    result: list[str] = []
+
+    for line in lines:
+        # 1. Consecutive duplicate
+        if collapse_repeats and line == prev_line and line.strip():
+            result.append("")
+            continue
+
+        # 2. Built-in noise (skip if line contains protected label)
+        stripped = line.strip()
+        if _is_builtin_noise(stripped):
+            if protected_substrings and any(p in line for p in protected_substrings):
+                pass  # keep
+            else:
+                result.append("")
+                prev_line = line
+                continue
+
+        # 3. Profile drop_lines rules
+        dropped = False
+        for pat in compiled_drop:
+            if pat.search(stripped):
+                if protected_substrings and any(p in line for p in protected_substrings):
+                    break
+                dropped = True
+                break
+        if dropped:
+            result.append("")
+            prev_line = line
+            continue
+
+        # 4. Inline drop patterns
+        for pat in compiled_inline:
+            line = pat.sub("", line)
+
+        # 5. Mask patterns
+        for pat, repl in compiled_mask:
+            line = pat.sub(repl, line)
+
+        result.append(line)
+        prev_line = line
+
+    return "\n".join(result)
+
+
+def denoise_stage(ctx: PipelineContext) -> None:
+    """Pipeline stage 3: denoise text."""
+    profile = ctx.profile
+    noise = profile.noise if profile else None
+
+    protected: set[str] = set()
+    if profile:
+        for field in profile.fields:
+            for alias in field.aliases:
+                protected.add(alias)
+
+    # Build drop_line_patterns from DropLinesRule
+    drop_line_patterns: list[str] = []
+    if noise and noise.drop_lines:
+        dl = noise.drop_lines
+        drop_line_patterns.extend(dl.regex)
+        # contains_any patterns → convert to regex
+        for phrase in dl.contains_any:
+            drop_line_patterns.append(re.escape(phrase))
+
+    text = ctx.normalized_text or ctx.raw_text
+    before_lines = [ln for ln in text.splitlines() if ln.strip()]
+    ctx.normalized_text = denoise(
+        text,
+        drop_line_patterns=drop_line_patterns,
+        drop_inline_patterns=list(noise.drop_patterns) if noise else [],
+        mask_patterns=list(noise.mask_patterns) if noise else [],
+        collapse_repeats=noise.collapse_repeats if noise else False,
+        protected_substrings=protected,
+    )
+    after_lines = [ln for ln in ctx.normalized_text.splitlines() if ln.strip()]
+    dropped = len(before_lines) - len(after_lines)
+    logger.debug(
+        "Denoiser: %d non-empty lines → %d (dropped %d, %d drop-patterns, %d mask-patterns)",
+        len(before_lines),
+        len(after_lines),
+        dropped,
+        len(drop_line_patterns),
+        len(noise.mask_patterns) if noise else 0,
+    )

ocr_postprocess/engine/extractor_stage.py

@@ -0,0 +1,107 @@
+"""Stage 5 — Extractor dispatcher."""
+
+from __future__ import annotations
+
+import logging
+
+from ocr_postprocess.models import Candidate, PipelineContext
+
+logger = logging.getLogger(__name__)
+
+
+def extract_stage(ctx: PipelineContext) -> None:
+    """Pipeline stage 5: run extractors for all profile fields."""
+    from ocr_postprocess.extractors import registry
+
+    profile = ctx.profile
+    if not profile:
+        logger.warning("No profile loaded; skipping extraction")
+        return
+
+    # Auto-import all registered extractors
+    _ensure_extractors_imported()
+
+    candidates: list[Candidate] = []
+
+    # Structured extractors first
+    failed_count = 0
+    for struct_ref in profile.structured_extractors:
+        try:
+            extractor = registry.get_instance(struct_ref.name)
+            cands = extractor.extract(ctx, field=None)
+            candidates.extend(cands)
+        except Exception:
+            failed_count += 1
+            logger.exception("Structured extractor '%s' failed", struct_ref.name)
+
+    # Per-field extractors
+    for field in profile.fields:
+        if field.constant is not None:
+            # Constant — skip extraction, reconciler will inject
+            continue
+
+        if field.compute is not None:
+            # Compute — skip extraction, transform stage handles
+            continue
+
+        if not field.extractor:
+            continue
+
+        try:
+            extractor = registry.get_instance(field.extractor)
+            cands = extractor.extract(ctx, field=field)
+            for c in cands:
+                c.key = field.key
+            candidates.extend(cands)
+        except Exception:
+            failed_count += 1
+            logger.exception("Extractor '%s' for field '%s' failed", field.extractor, field.key)
+
+        # Default fallback
+        if not any(c.key == field.key for c in candidates) and field.default is not None:
+            candidates.append(
+                Candidate(
+                    key=field.key,
+                    value=field.default,
+                    raw=str(field.default),
+                    extractor="default",
+                    sources=["default"],
+                    confidence=0.3,
+                    needs_llm_review=True,
+                )
+            )
+
+    ctx.candidates = candidates
+    if failed_count:
+        logger.warning(
+            "Extractor stage: %d candidate(s) from %d field(s); %d extractor(s) failed",
+            len(candidates),
+            len(profile.fields),
+            failed_count,
+        )
+    else:
+        logger.debug(
+            "Extractor stage: %d candidate(s) from %d field(s)",
+            len(candidates),
+            len(profile.fields),
+        )
+
+
+def _ensure_extractors_imported() -> None:
+    """Import all extractor submodules so @register decorators run."""
+    import importlib
+    import pkgutil
+
+    import ocr_postprocess.extractors.label_anchor as la_pkg
+    import ocr_postprocess.extractors.pattern as pat_pkg
+    import ocr_postprocess.extractors.structured as st_pkg
+
+    for pkg in (pat_pkg, la_pkg, st_pkg):
+        pkg_path = pkg.__path__
+        pkg_name = pkg.__name__
+        for _finder, mod_name, _ispkg in pkgutil.iter_modules(pkg_path):
+            full_name = f"{pkg_name}.{mod_name}"
+            try:
+                importlib.import_module(full_name)
+            except Exception:
+                logger.debug("Could not import %s", full_name)

ocr_postprocess/engine/normalizer.py

@@ -0,0 +1,128 @@
+"""Stage 1 — Normalizer: Unicode, whitespace, OCR fixes, dedup repeats."""
+
+from __future__ import annotations
+
+import logging
+import unicodedata
+
+import regex as re
+
+from ocr_postprocess.models import PipelineContext
+
+logger = logging.getLogger(__name__)
+
+# Maps common OCR letter-for-digit misreads: O→0, l→1, S→5
+_OCR_DIGIT_MAP = str.maketrans("OlS", "015")
+
+
+def _fix_ocr_in_digit_tokens(text: str) -> str:
+    """Replace common OCR letter mistakes inside all-digit-like tokens."""
+
+    def fix_token(m: re.Match) -> str:
+        token = m.group(0)
+        # Only fix tokens that look like numbers mixed with OCR errors
+        candidate = token.translate(_OCR_DIGIT_MAP)
+        if candidate.isdigit() and not token.isdigit():
+            return candidate
+        return token
+
+    return re.sub(r"\b[0-9OlS]{2,}\b", fix_token, text)
+
+
+def _dedup_adjacent_repeats(text: str, min_len: int = 5, max_len: int = 30) -> str:
+    """Remove directly adjacent repeated substrings on the same line."""
+    lines = text.split("\n")
+    result = []
+    for line in lines:
+        if len(line) < min_len * 2:
+            result.append(line)
+            continue
+        # Try windows from largest to smallest
+        changed = True
+        while changed:
+            changed = False
+            for length in range(min(max_len, len(line) // 2), min_len - 1, -1):
+                i = 0
+                while i + length * 2 <= len(line):
+                    chunk = line[i : i + length]
+                    if line[i + length : i + length * 2] == chunk:
+                        # Remove the first occurrence
+                        line = line[:i] + line[i + length :]
+                        changed = True
+                        break
+                    i += 1
+                if changed:
+                    break
+        result.append(line)
+    return "\n".join(result)
+
+
+def normalize(
+    text: str,
+    ocr_fixes: bool = True,
+    dedup_repeats: bool = True,
+    preserve_multi_space: bool = True,
+) -> str:
+    """Apply full normalization pipeline to raw OCR text."""
+    # 1. Unicode NFC
+    text = unicodedata.normalize("NFC", text)
+
+    # 2. Strip control chars (keep \n \t)
+    text = "".join(ch for ch in text if ch in ("\n", "\t") or (ord(ch) >= 0x20 and ord(ch) != 0x7F))
+
+    # 3. Whitespace normalize
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    text = text.replace("\t", "    ")
+
+    lines = text.split("\n")
+    normalized_lines = []
+    for line in lines:
+        # Trim trailing space
+        line = line.rstrip()
+        # Collapse multiple spaces (unless preserve_multi_space)
+        if not preserve_multi_space:
+            line = re.sub(r" {2,}", " ", line)
+        normalized_lines.append(line)
+    text = "\n".join(normalized_lines)
+
+    # 4. Smart quotes
+    text = (
+        text.replace("\u2018", "'")
+        .replace("\u2019", "'")
+        .replace("\u201c", '"')
+        .replace("\u201d", '"')
+    )
+
+    # 5. OCR digit fixes
+    if ocr_fixes:
+        text = _fix_ocr_in_digit_tokens(text)
+
+    # 6. Dedup adjacent repeats
+    if dedup_repeats:
+        text = _dedup_adjacent_repeats(text)
+
+    return text
+
+
+def normalize_stage(ctx: PipelineContext) -> None:
+    """Pipeline stage 1: normalize raw text."""
+    profile = ctx.profile
+    cfg = {}
+    if profile and profile.normalize:
+        cfg = profile.normalize
+
+    ctx.normalized_text = normalize(
+        ctx.raw_text,
+        ocr_fixes=cfg.get("ocr_fixes", True),
+        dedup_repeats=cfg.get("dedup_repeats", True),
+        preserve_multi_space=cfg.get("preserve_multi_space", True),
+    )
+    in_lines = ctx.raw_text.count("\n") + 1
+    out_lines = ctx.normalized_text.count("\n") + 1
+    logger.debug(
+        "Normalizer: %d → %d chars, %d → %d lines",
+        len(ctx.raw_text),
+        len(ctx.normalized_text),
+        in_lines,
+        out_lines,
+    )