ocr-postprocess 0.1.0__py3-none-any.whl

ocr_postprocess/profiles/loader.py

@@ -0,0 +1,63 @@
+"""Profile YAML loader."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+import yaml
+from pydantic import ValidationError
+
+from ocr_postprocess.exceptions import ProfileValidationError
+from ocr_postprocess.profiles.schema import DocumentProfile
+
+logger = logging.getLogger(__name__)
+
+
+def load_profile(path: Path) -> DocumentProfile:
+    """Load and validate a single YAML profile file."""
+    try:
+        raw = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        raise ProfileValidationError(f"YAML parse error: {exc}", field_path=str(path))
+
+    if not isinstance(raw, dict):
+        raise ProfileValidationError("Profile must be a YAML mapping", field_path=str(path))
+
+    # Use filename stem as id if not provided
+    if "id" not in raw:
+        raw["id"] = path.stem
+
+    try:
+        profile = DocumentProfile.model_validate(raw)
+    except ValidationError as exc:
+        errors = "; ".join(
+            f"{'.'.join(str(loc) for loc in e['loc'])}: {e['msg']}" for e in exc.errors()
+        )
+        raise ProfileValidationError(errors, field_path=str(path))
+
+    # Validate id matches filename
+    if profile.id != path.stem:
+        logger.warning("Profile id '%s' does not match filename '%s'", profile.id, path.stem)
+
+    logger.debug("Loaded profile '%s' from %s", profile.id, path)
+    return profile
+
+
+def load_profiles(profiles_dir: str | Path) -> dict[str, DocumentProfile]:
+    """Load all YAML profiles from a directory."""
+    profiles_path = Path(profiles_dir)
+    if not profiles_path.is_dir():
+        logger.warning("Profiles directory not found: %s", profiles_path)
+        return {}
+
+    profiles: dict[str, DocumentProfile] = {}
+    for yml_file in sorted(profiles_path.glob("*.yml")):
+        try:
+            profile = load_profile(yml_file)
+            profiles[profile.id] = profile
+        except ProfileValidationError as exc:
+            logger.error("Failed to load profile %s: %s", yml_file.name, exc)
+
+    logger.info("Loaded %d profile(s) from %s", len(profiles), profiles_path)
+    return profiles

ocr_postprocess/profiles/matcher.py

@@ -0,0 +1,71 @@
+"""Classify expression evaluator (AND/OR/NOT tree)."""
+
+from __future__ import annotations
+
+import logging
+import re
+
+from rapidfuzz import fuzz
+
+from ocr_postprocess.profiles.schema import ClassifyExpr
+
+logger = logging.getLogger(__name__)
+
+
+def _contains_any(text: str, keywords: list[str], threshold: float) -> float:
+    """Return max fuzzy partial_ratio score across keywords, or 0."""
+    best = 0.0
+    text_lower = text.lower()
+    for kw in keywords:
+        score = fuzz.partial_ratio(kw.lower(), text_lower) / 100.0
+        if score >= threshold:
+            best = max(best, score)
+    return best
+
+
+def _regex_match(text: str, pattern: str) -> float:
+    """Return 1.0 if pattern matches, else 0.0."""
+    return 1.0 if re.search(pattern, text) else 0.0
+
+
+def evaluate(expr: ClassifyExpr | str, text: str, threshold: float | None = None) -> float:
+    """Recursively evaluate a ClassifyExpr against text.
+
+    Returns score in [0, 1]:
+      - 0 means no match / failed condition
+      - 1 means perfect match
+    """
+    if isinstance(expr, str):
+        # leaf — keyword
+        score = fuzz.partial_ratio(expr.lower(), text.lower()) / 100.0
+        return score if score >= (threshold or 0.85) else 0.0
+
+    thr = threshold if threshold is not None else expr.fuzzy_threshold
+
+    # contains_any leaf
+    if expr.contains_any is not None:
+        return _contains_any(text, expr.contains_any, thr)
+
+    # regex leaf
+    if expr.regex is not None:
+        return _regex_match(text, expr.regex)
+
+    # NOT
+    if expr.not_ is not None:
+        inner = evaluate(expr.not_, text, thr)
+        return 0.0 if inner > 0.0 else 1.0
+
+    # AND
+    if expr.all_of is not None:
+        scores = [evaluate(child, text, thr) for child in expr.all_of]
+        if any(s == 0.0 for s in scores):
+            return 0.0
+        return sum(scores) / len(scores)
+
+    # OR
+    if expr.any_of is not None:
+        scores = [evaluate(child, text, thr) for child in expr.any_of]
+        positives = [s for s in scores if s > 0.0]
+        return max(positives) if positives else 0.0
+
+    return 0.0

ocr_postprocess/profiles/schema.py

@@ -0,0 +1,197 @@
+"""Profile YAML schema — Pydantic v2 models."""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class ClassifyExpr(BaseModel):
+    """AND/OR/NOT expression tree for classifier."""
+
+    all_of: list["ClassifyExpr | str"] | None = None
+    any_of: list["ClassifyExpr | str"] | None = None
+    not_: "ClassifyExpr | str | None" = Field(default=None, alias="not")
+    contains_any: list[str] | None = None
+    regex: str | None = None
+    fuzzy_threshold: float = 0.85
+
+    model_config = {"populate_by_name": True}
+
+    @model_validator(mode="after")
+    def at_least_one_condition(self) -> "ClassifyExpr":
+        has = any(
+            [
+                self.all_of is not None,
+                self.any_of is not None,
+                self.not_ is not None,
+                self.contains_any is not None,
+                self.regex is not None,
+            ]
+        )
+        if not has:
+            raise ValueError("ClassifyExpr must have at least one condition")
+        return self
+
+
+class DropLinesRule(BaseModel):
+    """Rules for which lines to drop during denoising."""
+
+    regex: list[str] = []
+    contains_any: list[str] = []
+
+
+class NoiseRules(BaseModel):
+    """Noise-removal rules applied during denoising stage."""
+
+    drop_lines: DropLinesRule = Field(default_factory=DropLinesRule)
+    drop_patterns: list[str] = []
+    mask_patterns: list[dict[str, str]] = []
+    collapse_repeats: bool = False
+
+    @model_validator(mode="before")
+    @classmethod
+    def _normalise_drop_lines(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            data = dict(data)
+            dl = data.get("drop_lines")
+            if isinstance(dl, list):
+                data["drop_lines"] = {"regex": [], "contains_any": dl}
+        return data
+
+
+class SectionDef(BaseModel):
+    """Definition of a document section and how to detect its start."""
+
+    id: str
+    start: list[str]
+    is_regex: bool = False
+
+
+class ReconstructConfig(BaseModel):
+    """Configuration for the reconstructor stage (6 sub-steps a–f)."""
+
+    enabled_steps: list[str] = ["a", "b", "c", "d", "e", "f"]
+    bilingual_separator: str = "/"
+    fuzzy_threshold: float = 0.8
+    multi_label_min_count: int = 2
+    split_glued_labels: bool = True
+    rejoin_wrapped_lines: bool = True
+    bilingual_pairs: list[list[str]] = []
+
+
+class TransformOpRef(BaseModel):
+    """One op in a transform pipeline."""
+
+    name: str
+    args: dict[str, Any] = {}
+
+
+class FieldDef(BaseModel):
+    """Definition of one extractable field in a document profile."""
+
+    model_config = {"populate_by_name": True}
+
+    key: str = Field(default="", alias="name")
+    extractor: str | None = None
+    aliases: list[str] = []
+    section: str | None = None
+    pattern: str | None = None
+    transform: list[Any] = []
+    compute: str | None = None
+    deps: list[str] = []
+    constant: Any = None
+    default: Any = None
+    optional: bool = False
+    required: bool = False
+    type: Literal["text", "int", "float", "date", "checkbox", "enum"] = "text"
+    enum_values: list[str] | None = None
+    needs_vision: bool = False
+    llm_hint: str | None = None
+    cross_validate_with: list[str] = Field(default_factory=list)
+    fuzzy_label: bool = True
+    confidence_threshold: float = 0.0
+    extractor_args: dict[str, Any] = {}
+    stop_labels: list[str] = []
+    next_lines: int = 0  # Extra lines to search after the label line (used by regex_after_label)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _remap_aliases(cls, data: Any) -> Any:
+        """Accept YAML field names that differ from model field names."""
+        if isinstance(data, dict):
+            data = dict(data)
+            # cross_check_with → cross_validate_with
+            if "cross_check_with" in data and "cross_validate_with" not in data:
+                data["cross_validate_with"] = data.pop("cross_check_with")
+            # Ensure key is populated if name was given
+            if "name" in data and "key" not in data:
+                data["key"] = data["name"]
+        return data
+
+
+class StructuredExtractorRef(BaseModel):
+    """Reference to a structured extractor with optional args."""
+
+    name: str
+    section: str | None = None
+    args: dict[str, Any] = {}
+
+
+class OutputConfig(BaseModel):
+    """Output format configuration (markdown, JSON)."""
+
+    markdown: dict[str, Any] = {}
+    json_output: dict[str, Any] = Field(default_factory=dict, alias="json")
+
+
+class DocumentProfile(BaseModel):
+    """Complete document profile: classification rules, fields, and pipeline config."""
+
+    id: str
+    name: str = ""
+    display_name: str = ""
+    version: int = 1
+    language: list[str] = ["vi"]
+    extends: str | None = None
+
+    classify: ClassifyExpr
+    normalize: dict[str, Any] = {}
+    noise: NoiseRules = Field(default_factory=NoiseRules)
+    sections: list[SectionDef] = []
+    reconstruct: ReconstructConfig = Field(default_factory=ReconstructConfig)
+
+    fields: list[FieldDef] = []
+    extract: list[FieldDef] = []  # YAML uses 'extract', normalised to fields
+    structured_extractors: list[StructuredExtractorRef] = []
+    compute: list[dict[str, Any]] = []
+
+    output: OutputConfig = Field(default_factory=OutputConfig)
+
+    model_config = {"populate_by_name": True}
+
+    @model_validator(mode="before")
+    @classmethod
+    def _remap_yaml_keys(cls, data: Any) -> Any:
+        """Map YAML-style keys to schema field names."""
+        if isinstance(data, dict):
+            data = dict(data)
+            # denoise → noise
+            if "denoise" in data and "noise" not in data:
+                data["noise"] = data.pop("denoise")
+        return data
+
+    @model_validator(mode="after")
+    def normalise_fields(self) -> "DocumentProfile":
+        # Merge 'extract' into 'fields'
+        if self.extract and not self.fields:
+            self.fields = self.extract
+        return self
+
+    def get_field(self, key: str) -> FieldDef | None:
+        """Look up a field by key."""
+        for f in self.fields:
+            if f.key == key:
+                return f
+        return None

ocr_postprocess/renderer/json_renderer.py

@@ -0,0 +1,59 @@
+"""JSON renderer for ProcessedDocument — output-format.md §9.2."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ocr_postprocess.models import ProcessedDocument
+
+
+def _build_output(doc: ProcessedDocument) -> dict[str, Any]:
+    """Build structured dict matching the output-format.md JSON schema."""
+    return {
+        "profile": {
+            "id": doc.profile_id,
+            "score": round(doc.profile_score, 4),
+        },
+        "sections": [
+            {
+                "id": s.id,
+                "title": s.title,
+                "lines": [line.text for line in s.lines],
+            }
+            for s in doc.sections
+        ],
+        "candidates": [
+            {
+                "key": c.key,
+                "value": c.value,
+                "raw": c.raw,
+                "extractor": c.extractor,
+                "sources": c.sources,
+                "section_id": c.section_id,
+                "confidence": round(c.confidence, 4),
+                "needs_llm_review": c.needs_llm_review,
+                "needs_vision": c.needs_vision,
+                "conflict": c.conflict,
+                "notes": c.notes,
+            }
+            for c in doc.candidates
+        ],
+        "hints": doc.hints,
+        "cross_checks": [
+            {
+                "field_key": cc.field_key,
+                "sources": cc.sources,
+                "matched": cc.matched,
+                "detail": cc.detail,
+            }
+            for cc in doc.cross_checks
+        ],
+        "warnings": doc.warnings,
+        "overall_confidence": round(doc.overall_confidence, 4),
+    }
+
+
+def to_json(doc: ProcessedDocument, indent: int = 2) -> str:
+    """Serialize ProcessedDocument to JSON string (output-format.md schema)."""
+    return json.dumps(_build_output(doc), ensure_ascii=False, indent=indent)

ocr_postprocess/renderer/llm.py

@@ -0,0 +1,41 @@
+"""Renderer: LLM-friendly Markdown output — clean key-value table for use as LLM context."""
+
+from __future__ import annotations
+
+from ocr_postprocess.models import ProcessedDocument
+
+
+def render_llm_markdown(doc: ProcessedDocument) -> str:
+    """Render ProcessedDocument thành Markdown gọn cho LLM.
+
+    Chỉ hiển thị bảng key-value sạch (không có extractor metadata, cross-checks,
+    hay confidence scores) — phù hợp để gửi vào context của LLM.
+    """
+    lines: list[str] = []
+
+    title = doc.profile_display_name or doc.profile_id
+    lines.append(f"# {title}")
+    lines.append("")
+    lines.append("| Trường | Giá trị |")
+    lines.append("|--------|---------|")
+
+    # Deduplicate: chỉ lấy candidate đầu tiên của mỗi key (đã qua reconcile)
+    seen_keys: set[str] = set()
+    for cand in doc.candidates:
+        if cand.key in seen_keys:
+            continue
+        seen_keys.add(cand.key)
+        label = doc.field_labels.get(cand.key) or cand.key
+        value = str(cand.value) if cand.value is not None else ""
+        if value:
+            lines.append(f"| {label} | {value} |")
+
+    lines.append("")
+
+    if doc.warnings:
+        lines.append("> **Lưu ý:**")
+        for w in doc.warnings:
+            lines.append(f"> - {w}")
+        lines.append("")
+
+    return "\n".join(lines)

ocr_postprocess/renderer/markdown.py

@@ -0,0 +1,172 @@
+"""Renderer: Markdown output for ProcessedDocument — Stage 9."""
+
+from __future__ import annotations
+
+import logging
+
+from ocr_postprocess.models import Candidate, PipelineContext, ProcessedDocument
+
+logger = logging.getLogger(__name__)
+
+
+def _meta(cand: Candidate) -> str:
+    """Build «extractor [flags]» meta string."""
+    if cand.extractor in ("constant", "computed"):
+        base = f"({cand.extractor})"
+    else:
+        base = cand.extractor
+        if any("checksum" in s for s in cand.sources):
+            base += " ✓ checksum"
+
+    flags = []
+    if cand.needs_llm_review:
+        flags.append("?review")
+    if cand.needs_vision:
+        flags.append("?vision")
+    if cand.conflict:
+        flags.append("!conflict")
+
+    tail = " ".join(flags)
+    inner = (base + " " + tail).strip() if tail else base
+    return f"«{inner}»"
+
+
+def render_markdown(doc: ProcessedDocument) -> str:
+    """Render ProcessedDocument to Markdown per output-format.md spec."""
+    lines: list[str] = []
+    conf = doc.overall_confidence
+    title = doc.profile_display_name or doc.profile_id or "unknown"
+    lines.append(f"# {title} (confidence {conf:.2f})")
+    lines.append("")
+
+    # Group candidates by section
+    section_candidates: dict[str, list[Candidate]] = {}
+    unplaced: list[Candidate] = []
+    for cand in doc.candidates:
+        if cand.section_id:
+            section_candidates.setdefault(cand.section_id, []).append(cand)
+        else:
+            unplaced.append(cand)
+
+    # Per-section listings — value only, no extractor metadata (avoids confusing LLM)
+    for section in doc.sections:
+        cands = section_candidates.get(section.id, [])
+        if not cands:
+            continue
+        title_part = f" ({section.title})" if section.title else ""
+        lines.append(f"## Section: {section.id}{title_part}")
+        for cand in cands:
+            value_str = str(cand.value) if cand.value is not None else "—"
+            label = doc.field_labels.get(cand.key) or cand.key
+            lines.append(f"- **{label}**: {value_str}")
+        lines.append("")
+
+    # Fallback table for unplaced candidates
+    if unplaced:
+        lines.append("## Extracted Fields")
+        lines.append("")
+        lines.append("| Field | Value | Extractor | Confidence | Flags |")
+        lines.append("|-------|-------|-----------|------------|-------|")
+        for cand in unplaced:
+            value_str = str(cand.value) if cand.value is not None else "—"
+            conf_str = f"{cand.confidence:.2f}"
+            flag_parts = []
+            if cand.needs_llm_review:
+                flag_parts.append("?review")
+            if cand.conflict:
+                flag_parts.append("!conflict")
+            if cand.needs_vision:
+                flag_parts.append("?vision")
+            flag_str = " ".join(flag_parts)
+            lines.append(
+                f"| `{cand.key}` | {value_str} | {_meta(cand)} | {conf_str} | {flag_str} |"
+            )
+        lines.append("")
+
+    # Hints
+    if doc.hints:
+        lines.append("## Hints")
+        for hint_key, hint_vals in doc.hints.items():
+            lines.append(f"- {hint_key}: {hint_vals}")
+        lines.append("")
+
+    # Lưu ý — gộp conflict warnings + failed cross-checks thành một section duy nhất.
+    # Mỗi dòng chỉ rõ giá trị đã chọn + nguồn + confidence để LLM có đủ context suy luận.
+    chosen: dict[str, str] = {c.key: str(c.value) for c in doc.candidates if c.value is not None}
+    failed_checks = {cc.field_key: cc for cc in doc.cross_checks if not cc.matched}
+    missing = [w for w in doc.warnings if w.startswith("Required field missing")]
+
+    # Helper: extract ±context_lines lines around occurrences of raw strings in canonical_text
+    ocr_lines: list[str] = doc.canonical_text.splitlines() if doc.canonical_text else []
+
+    def _snippet(anchor_raw: str, context: int = 2) -> str:
+        """Return lines (±context) around the first line containing anchor_raw."""
+        if not ocr_lines or not anchor_raw:
+            return ""
+        for i, ln in enumerate(ocr_lines):
+            if anchor_raw in ln:
+                lo = max(0, i - context)
+                hi = min(len(ocr_lines) - 1, i + context)
+                return "\n".join(ocr_lines[lo : hi + 1])
+        return ""
+
+    notes: list[str] = []
+    for field_key, cc in failed_checks.items():
+        label = doc.field_labels.get(field_key)
+        # Skip internal/technical fields that have no human-readable label
+        if not label:
+            continue
+        selected = chosen.get(field_key)
+        # Build value→{extractor, confidence, raw, line_index} from value_sources
+        src_map: dict[str, dict] = {vs["value"]: vs for vs in cc.value_sources}
+
+        def _fmt(val: str) -> str:
+            vs = src_map.get(val, {})
+            ext = vs.get("extractor", "?")
+            conf = vs.get("confidence")
+            raw = vs.get("raw", "")
+            conf_str = f", {conf:.2f}" if conf is not None else ""
+            raw_str = f", raw: `{raw}`" if raw and raw != val else ""
+            return f'**"{val}"** ({ext}{conf_str}{raw_str})'
+
+        # Anchor snippet on selected value's raw text (most relevant context)
+        anchor = src_map.get(selected or "", {}).get("raw", "") if selected else ""
+        if not anchor and src_map:
+            anchor = next(iter(src_map.values())).get("raw", "")
+        snippet = _snippet(anchor)
+
+        if selected:
+            alternatives = [v for v in cc.values if v != selected]
+            if alternatives:
+                alt_parts = " | ".join(_fmt(v) for v in alternatives)
+                note = (
+                    f"⚠️ **{label}**: hệ thống chọn {_fmt(selected)}"
+                    f" — cũng nhận dạng được: {alt_parts}"
+                )
+                if snippet:
+                    note += f"\n\n  ```\n  {chr(10).join('  ' + ln for ln in snippet.splitlines())}\n  ```"
+                notes.append(note)
+        elif cc.values:
+            parts = " | ".join(_fmt(v) for v in cc.values)
+            note = f"⚠️ **{label}**: không thể xác định chắc chắn — tìm thấy: {parts}"
+            if snippet:
+                note += (
+                    f"\n\n  ```\n  {chr(10).join('  ' + ln for ln in snippet.splitlines())}\n  ```"
+                )
+            notes.append(note)
+    for w in missing:
+        notes.append(f"⚠️ {w}")
+
+    if notes:
+        lines.append("## Lưu ý")
+        lines.append("")
+        for note in notes:
+            lines.append(f"- {note}")
+            lines.append("")
+
+    return "\n".join(lines)
+
+
+def render_stage(ctx: PipelineContext) -> None:
+    """Pipeline stage 9: no-op hook — markdown built in pipeline.process()."""
+    logger.debug("Render stage: markdown will be assembled after all stages complete")

ocr_postprocess/scorer.py

@@ -0,0 +1,78 @@
+"""Stage 8 — ConfidenceScorer."""
+
+from __future__ import annotations
+
+import logging
+
+from ocr_postprocess.models import PipelineContext
+
+logger = logging.getLogger(__name__)
+
+
+def score_stage(ctx: PipelineContext) -> None:
+    """Pipeline stage 8: compute overall_confidence."""
+    profile = ctx.profile
+
+    # Classification score
+    classification_score = ctx.classification_score
+
+    # % required fields found
+    if profile and profile.fields:
+        required_fields = [f for f in profile.fields if f.required]
+        if required_fields:
+            found_keys = {c.key for c in ctx.candidates}
+            pct_required = sum(1 for f in required_fields if f.key in found_keys) / len(
+                required_fields
+            )
+        else:
+            pct_required = 1.0
+
+        # avg field confidence
+        if ctx.candidates:
+            avg_conf = sum(c.confidence for c in ctx.candidates) / len(ctx.candidates)
+        else:
+            avg_conf = 0.0
+
+        # % cross-validated
+        if ctx.cross_checks:
+            pct_cross = sum(1 for cc in ctx.cross_checks if cc.matched) / len(ctx.cross_checks)
+        else:
+            pct_cross = 1.0
+    else:
+        pct_required = 1.0
+        avg_conf = sum(c.confidence for c in ctx.candidates) / max(len(ctx.candidates), 1)
+        pct_cross = 1.0
+
+    overall = (
+        # classification_score: 20% — reward correct document type identification
+        classification_score * 0.2
+        # pct_required: 30% — largest weight, all required fields must be found
+        + pct_required * 0.3
+        # avg_conf:     30% — average extractor confidence across all candidates
+        + avg_conf * 0.3
+        # pct_cross:    20% — reward consistency between independent extractors
+        + pct_cross * 0.2
+    )
+
+    # Penalty for missing required fields: -0.2 per missing field (capped at 0.0)
+    missing_required = [
+        f.key
+        for f in (profile.fields if profile else [])
+        if f.required and not any(c.key == f.key for c in ctx.candidates)
+    ]
+    if missing_required:
+        overall = max(0.0, overall - 0.2 * len(missing_required))
+        # Reconciler already appended warnings; log here for scorer-level visibility
+        for key in missing_required:
+            logger.warning("Scorer: required field still missing after reconcile: '%s'", key)
+
+    ctx.overall_confidence = round(min(1.0, max(0.0, overall)), 4)
+    logger.debug(
+        "Scorer: classify=%.2f required=%.2f avg_conf=%.2f cross=%.2f → overall=%.4f%s",
+        classification_score,
+        pct_required,
+        avg_conf,
+        pct_cross,
+        ctx.overall_confidence,
+        f" (missing required: {missing_required})" if missing_required else "",
+    )