paradox2 0.1.0__py3-none-any.whl

paradox2/__init__.py

@@ -0,0 +1,7 @@
+"""Public package surface for the from-scratch extractor."""
+
+from paradox2.api import extract, extract_kie, extract_tables, extract_text, read
+from paradox2.doctor import doctor
+from paradox2.runtime.machine_state import warmup
+
+__all__ = ["extract", "extract_kie", "extract_tables", "extract_text", "read", "doctor", "warmup"]

paradox2/__main__.py

@@ -0,0 +1,5 @@
+"""Allow ``python -m paradox2`` as well as the installed console script."""
+
+from paradox2.cli import main
+
+raise SystemExit(main())

paradox2/api.py

@@ -0,0 +1,320 @@
+"""Public facade — dispatch only, no business logic.
+
+The old api.py grew to 2,783 lines by letting business logic (add-on
+dispatch, backend resolution details, overlay rendering) live in the
+facade itself. This file must stay thin: routing to pipeline.* and
+attaching tables. If this file starts approaching 100+ lines, that's a
+signal work leaked out of pipeline/ and should be pushed back down
+(see docs/PLAN.md Phase 2).
+"""
+
+from __future__ import annotations
+
+import pymupdf
+import re
+from pathlib import Path
+from collections.abc import Sequence
+
+from paradox2.pipeline import digital_fast, scanned
+from paradox2.pipeline.digital_fast import PageResult
+from paradox2.tables.line_tables import detect_table_bboxes
+
+
+def _page_selection(pages) -> set[int] | None:
+    if pages is None:
+        return None
+    if isinstance(pages, int):
+        return {pages - 1}
+    if isinstance(pages, str):
+        selected: set[int] = set()
+        for part in pages.replace(" ", "").split(","):
+            if not part:
+                continue
+            if "-" in part:
+                start, end = part.split("-", 1)
+                selected.update(range(int(start) - 1, int(end)))
+            else:
+                selected.add(int(part) - 1)
+        return selected
+    return {int(page) - 1 for page in pages}
+
+
+def extract(
+    path: str,
+    backend: str = "auto",
+    language: str = "latin",
+    *,
+    pages: Sequence[int] | str | int | None = None,
+    feature: str | list[str] | None = None,
+    fields: bool = False,
+    include_links: bool = False,
+    include_annotations: bool = False,
+    detect_language: bool = False,
+    output_format: str = "dict",
+    mode: str | None = None,
+    config=None,
+    classify: bool = False,
+    handwriting: bool = False,
+    signatures: bool = False,
+    formulas: bool = False,
+    marks: bool = False,
+    detect_formulas: bool = False,
+    detect_signatures: bool = False,
+    page_workers: int = 1,
+):
+    """Extract every page of a PDF. Digital pages use the zero-ML fast path
+    (with vector-line table detection attached); scanned pages route to the
+    GPU OCR pipeline. Each page decides independently — a mixed digital+
+    scanned document is handled correctly per-page.
+
+    page_workers>1 parallelizes only the scanned pages across a
+    ProcessPoolExecutor (see pipeline.scanned.extract_scanned_parallel for
+    the measured numbers: ~1.9x on GPU, ~2x on CPU vs sequential, on a
+    12-page real document). Digital pages stay on the sequential zero-ML
+    path regardless — they're already fast enough that process-spawn
+    overhead would not pay for itself."""
+    min_native_text_chars = digital_fast._MIN_NATIVE_TEXT_CHARS
+    scanned_dpi = 200  # matches scanned.py's own _page_to_image default
+    if config is not None:
+        # Config is an explicit convenience object; direct keyword arguments
+        # remain the most local override for callers that need one-off changes.
+        backend = getattr(config, "backend", backend)
+        language = getattr(config, "language", language)
+        include_links = include_links or bool(getattr(config, "include_links", False))
+        include_annotations = include_annotations or bool(getattr(config, "include_annotations", False))
+        detect_language = detect_language or bool(getattr(config, "detect_language", False))
+        # Paradox-pdf 0.9.2 parity knob (was "scan_text_threshold") - only
+        # overrides the digital/scanned routing decision when a caller
+        # explicitly sets it via ExtractConfig; every existing caller keeps
+        # today's behavior unchanged (default matches the module constant).
+        min_native_text_chars = getattr(config, "scan_text_threshold", min_native_text_chars)
+        # ExtractConfig.dpi existed but was never read here - only affects scanned-OCR render.
+        scanned_dpi = getattr(config, "dpi", scanned_dpi)
+
+    # Keep non-PDF adapters behind the same public facade. They are already
+    # normalized to PageResult by formats.detector and never pay PDF/OCR costs.
+    if Path(path).suffix.lower() != ".pdf":
+        from paradox2.formats.detector import detect_and_extract
+        from paradox2.exporters.structured import select_feature, serialize
+
+        result = detect_and_extract(path, backend=backend, language=language)
+        if feature is None and output_format == "dict":
+            return result
+        value = select_feature(result, feature or "all")
+        return serialize(value, output_format)
+
+    from paradox2.pipeline.feature_flags import resolve_requested_features
+
+    flags = resolve_requested_features(
+        feature=feature, fields=fields, include_links=include_links,
+        include_annotations=include_annotations, detect_language=detect_language,
+        mode=mode, classify=classify, handwriting=handwriting, signatures=signatures,
+        formulas=formulas, marks=marks, detect_formulas=detect_formulas,
+        detect_signatures=detect_signatures,
+    )
+    selected = _page_selection(pages)
+
+    doc = pymupdf.open(path)
+    digital_fast.reject_if_encrypted(doc, path)
+    try:
+        precomputed_scanned: dict[int, PageResult] = {}
+        page_dicts: dict[int, dict] = {}
+        if page_workers > 1:
+            precomputed_scanned, page_dicts = scanned.precompute_scanned_pages(
+                path, doc, selected, backend, language, page_workers, min_native_text_chars, scanned_dpi
+            )
+
+        results: list[PageResult] = []
+        for page in doc:
+            if selected is not None and page.number not in selected:
+                continue
+            # Single get_text("dict") call, reused for both the has-text
+            # routing check and (on the digital path) the real extraction —
+            # was two separate PyMuPDF text-extraction passes per digital
+            # page (has_text_layer's get_text("words") + extract_page's own
+            # get_text("dict")), pure duplicate work with no benefit. Reuse
+            # the pre-pass's cached dict when page_workers>1 already parsed
+            # this page above, instead of parsing it a second time.
+            raw = page_dicts.get(page.number)
+            if raw is None:
+                raw = page.get_text("dict")
+            has_text = digital_fast.raw_has_text(raw, min_native_text_chars)
+            if has_text:
+                result = digital_fast.extract_page(page, raw=raw)
+                if flags.tables:
+                    # Stroke-line extraction is expensive on vector-dense
+                    # pages (verified: ~20s/call on a real page with
+                    # 29,121 drawings) - compute once and share it between
+                    # bbox detection and each bbox's structure extraction,
+                    # instead of recomputing per call.
+                    from paradox2.tables.line_tables import _extract_stroke_lines_from_drawings
+
+                    horiz_segs, vert_segs = _extract_stroke_lines_from_drawings(page)
+                    result.tables = detect_table_bboxes(page, horiz=horiz_segs, vert=vert_segs)
+                    if result.tables:
+                        from paradox2.tables.structure import extract_table_structured
+
+                        for bbox in result.tables:
+                            result.structured_tables.extend(
+                                extract_table_structured(
+                                    page, bbox, horiz_segs=horiz_segs, vert_segs=vert_segs
+                                )
+                            )
+            else:
+                if page.number in precomputed_scanned:
+                    result = precomputed_scanned[page.number]
+                else:
+                    result = scanned.extract_page(page, backend=backend, language=language, dpi=scanned_dpi)
+                if flags.tables:
+                    from paradox2.tables.scanned_router import route_scanned_tables
+
+                    result.structured_tables = route_scanned_tables(page, result.blocks)
+
+            if flags.fields:
+                from paradox2.engines.kie_heuristic import (
+                    detect_grid_regions,
+                    extract_key_values,
+                    extract_key_values_spatial,
+                    looks_like_form,
+                )
+
+                if has_text:
+                    words = page.get_text("words")
+                    lines = page.get_text().splitlines()
+                    if looks_like_form(lines) or any(":" in line for line in lines):
+                        result.fields = extract_key_values_spatial(
+                            words, table_boxes=detect_grid_regions(words)
+                        )
+                else:
+                    result.fields = extract_key_values([b.text for b in result.blocks])
+
+            if flags.links:
+                from paradox2.pipeline.links import extract_page_hyperlinks
+
+                result.links = extract_page_hyperlinks(page)
+            if flags.annotations:
+                from paradox2.pipeline.annotations import extract_page_annotations
+
+                result.annotations = extract_page_annotations(page)
+            if flags.handwriting or flags.signatures or flags.formulas:
+                _apply_specialists(
+                    page, result,
+                    handwriting=flags.handwriting,
+                    signatures=flags.signatures,
+                    formulas=flags.formulas,
+                )
+            if flags.marks:
+                result.marks = [
+                    {"bbox": list(block.bbox), "text": block.text,
+                     "bold": block.bold, "italic": block.italic}
+                    for block in result.blocks if block.bold or block.italic
+                ]
+            if flags.language:
+                from paradox2.pipeline.language_detect import detect_text_language
+
+                result.language = detect_text_language(result.text)
+            result.metadata = {"page_number": page.number + 1, "source": str(path)}
+            results.append(result)
+        if flags.classify and results:
+            from paradox2.engines.doc_classifier import get_engine
+
+            label, confidence = get_engine().classify("\n".join(p.text for p in results))
+            classification = {"label": label, "confidence": confidence}
+            for result in results:
+                result.classification = classification
+        # Default remains the efficient, geometry-preserving IR used by the
+        # service and existing callers. Feature/output requests are pure
+        # projections: extraction is never repeated.
+        if feature is None:
+            if output_format == "dict":
+                return results
+            if output_format == "markdown":
+                from paradox2.exporters.structured import serialize
+
+                return serialize(results, "markdown")
+        from paradox2.exporters.structured import select_feature, serialize
+
+        value = select_feature(results, feature or "all")
+        return serialize(value, output_format)
+    finally:
+        doc.close()
+
+
+def extract_kie(path: str) -> list[list[dict]]:
+    """Per-page key-value extraction. Only meaningful on digital, form-like
+    pages (invoices/forms) - scanned pages and prose pages return []. Kept
+    separate from extract() rather than a feature= flag: KIE's output shape
+    (list of key-value dicts) doesn't fit PageResult, forcing it in would
+    be the wrong abstraction just to have one entry point."""
+    from paradox2.engines.kie_heuristic import (
+        detect_grid_regions,
+        extract_key_values_spatial,
+        looks_like_form,
+    )
+
+    doc = pymupdf.open(path)
+    try:
+        results: list[list[dict]] = []
+        for page in doc:
+            if not digital_fast.has_text_layer(page):
+                results.append([])
+                continue
+            words = page.get_text("words")
+            lines = page.get_text().splitlines()
+            if not looks_like_form(lines):
+                results.append([])
+                continue
+            regions = detect_grid_regions(words)
+            results.append(extract_key_values_spatial(words, table_boxes=regions))
+        return results
+    finally:
+        doc.close()
+
+
+def _render_crop(page, bbox=None, dpi: int = 180):
+    """Render a page/crop once for opt-in vision specialists."""
+    pix = page.get_pixmap(dpi=dpi, clip=pymupdf.Rect(bbox) if bbox else None, alpha=False)
+    from PIL import Image
+    return Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
+
+
+def _apply_specialists(page, result: PageResult, *, handwriting: bool, signatures: bool, formulas: bool):
+    """Run explicitly requested heavy models; never touched on the hot path."""
+    scale = 72.0 / 180.0
+    if signatures:
+        from paradox2.engines.signature_detr import get_engine
+        image = _render_crop(page)
+        for item in get_engine().detect(image):
+            item["bbox"] = [round(v * scale, 2) for v in item["bbox"]]
+            item["page"] = result.page_number
+            result.signatures.append(item)
+    if handwriting:
+        from paradox2.engines.handwriting_trocr import get_engine
+        engine = get_engine()
+        # TrOCR is line-oriented; use extracted blocks as tight crops and cap
+        # work so an accidental request cannot turn into an unbounded scan.
+        for block in result.blocks[:64]:
+            text = engine.process(_render_crop(page, block.bbox))
+            if text:
+                result.handwriting.append({"bbox": list(block.bbox), "text": text})
+    if formulas:
+        from paradox2.engines.formula_latex import image_to_latex
+        for block in result.blocks:
+            if not re.search(r"(?:=|∑|∫|√|\^|_[{\d])", block.text):
+                continue
+            latex = image_to_latex(_render_crop(page, block.bbox))
+            if latex:
+                result.formulas.append({"bbox": list(block.bbox), "text": block.text, "latex": latex})
+
+
+def read(path: str, **kwargs):
+    """Compatibility alias for universal multi-format dispatch."""
+    return extract(path, **kwargs)
+
+
+def extract_text(path: str, **kwargs) -> str:
+    return extract(path, feature="text", **kwargs)
+
+
+def extract_tables(path: str, **kwargs):
+    return extract(path, feature="tables", **kwargs)

paradox2/cli.py

@@ -0,0 +1,113 @@
+"""paradox2 CLI: one-shot extraction, and serve/client subcommands wrapping
+service/ (the persistent worker). argparse, stdlib only — no new deps for
+a CLI wrapper around already-built pipeline/service code.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from paradox2.service.protocol import DEFAULT_SOCKET_PATH
+
+
+def _cmd_extract(args: argparse.Namespace) -> int:
+    from paradox2.formats.detector import detect_and_extract
+    from paradox2.exporters.structured import page_to_dict, select_feature, serialize
+
+    pages = detect_and_extract(args.path, backend=args.backend, language=args.language)
+
+    if args.feature:
+        value = select_feature(pages, args.feature.split(","))
+        print(serialize(value, args.format))
+    elif args.format == "json":
+        print(json.dumps([page_to_dict(p) for p in pages], ensure_ascii=False))
+    else:
+        from paradox2.exporters.markdown import to_markdown
+
+        print(to_markdown(pages))
+    return 0
+
+
+def _cmd_serve(args: argparse.Namespace) -> int:
+    from paradox2.service.server import serve
+
+    serve(args.socket)
+    return 0
+
+
+def _cmd_client(args: argparse.Namespace) -> int:
+    from paradox2.service.client import extract, is_server_running
+
+    if not is_server_running(args.socket):
+        print(f"error: no paradox2 service running at {args.socket} "
+              f"(start one with 'paradox2 serve')", file=sys.stderr)
+        return 1
+
+    pages = extract(args.path, backend=args.backend, language=args.language, socket_path=args.socket)
+    for p in pages:
+        print(p.text)
+    return 0
+
+
+def _cmd_doctor(args: argparse.Namespace) -> int:
+    from paradox2.doctor import diagnose
+
+    report = diagnose()
+    if args.format == "json":
+        print(json.dumps(report, sort_keys=True))
+    else:
+        for key, value in report.items():
+            print(f"{key}: {value}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="paradox2")
+    sub = parser.add_subparsers(dest="command")
+
+    p_extract = sub.add_parser("extract", help="one-shot extraction (default if no subcommand given)")
+    p_extract.add_argument("path")
+    p_extract.add_argument("--backend", default="auto", choices=["auto", "gpu", "cpu"])
+    p_extract.add_argument("--language", default="latin")
+    p_extract.add_argument("--format", default="markdown", choices=["markdown", "json"])
+    p_extract.add_argument("--feature", help="comma-separated projection (text,fields,tables,...)")
+    p_extract.set_defaults(func=_cmd_extract)
+
+    p_serve = sub.add_parser("serve", help="start the persistent worker")
+    p_serve.add_argument("--socket", default=DEFAULT_SOCKET_PATH)
+    p_serve.set_defaults(func=_cmd_serve)
+
+    p_client = sub.add_parser("client", help="extract via a running service (fast, warm)")
+    p_client.add_argument("path")
+    p_client.add_argument("--socket", default=DEFAULT_SOCKET_PATH)
+    p_client.add_argument("--backend", default="auto", choices=["auto", "gpu", "cpu"])
+    p_client.add_argument("--language", default="latin")
+    p_client.set_defaults(func=_cmd_client)
+
+    p_doctor = sub.add_parser("doctor", help="report PDF/OCR runtime health")
+    p_doctor.add_argument("--format", choices=["text", "json"], default="text")
+    p_doctor.set_defaults(func=_cmd_doctor)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+
+    # Bare `paradox2 <path>` (no subcommand) defaults to extract, for
+    # convenience — matches the "1-second script" ergonomics goal.
+    if argv and argv[0] not in ("extract", "serve", "client", "doctor", "-h", "--help"):
+        argv = ["extract", *argv]
+
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if not getattr(args, "func", None):
+        parser.print_help()
+        return 1
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

paradox2/config.py

@@ -0,0 +1,33 @@
+"""Small explicit configuration surface for stable routing knobs."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+
+
+@dataclass(frozen=True, slots=True)
+class ExtractConfig:
+    backend: str = "auto"
+    language: str = "latin"
+    dpi: int = 200
+    include_links: bool = False
+    include_annotations: bool = False
+    detect_language: bool = False
+    # Paradox-pdf 0.9.2 called this "scan_text_threshold" (same value, 50) -
+    # kept as a distinct name here since digital_fast.py's own constant is
+    # named _MIN_NATIVE_TEXT_CHARS; this field is the public override knob
+    # for it (api.py wires it through raw_has_text's min_native_text_chars).
+    scan_text_threshold: int = 50
+
+    def with_overrides(self, **values) -> "ExtractConfig":
+        allowed = {
+            "backend", "language", "dpi", "include_links",
+            "include_annotations", "detect_language", "scan_text_threshold",
+        }
+        unknown = set(values) - allowed
+        if unknown:
+            raise TypeError(f"unknown extraction options: {sorted(unknown)}")
+        return replace(self, **values)
+
+
+__all__ = ["ExtractConfig"]

paradox2/doctor.py

@@ -0,0 +1,74 @@
+"""Runtime health checks; never hide a backend mismatch."""
+
+from __future__ import annotations
+
+from paradox2.runtime import machine_state
+from paradox2.runtime.backend_detect import has_cuda
+from paradox2.runtime.guardrails import BackendMismatch, check_backend
+
+
+def diagnose() -> dict[str, object]:
+    """Return a side-effect-free report suitable for CI and the CLI."""
+    report: dict[str, object] = {"cuda": has_cuda()}
+    try:
+        import pymupdf
+
+        report["pymupdf"] = getattr(pymupdf, "__version__", "installed")
+    except Exception as exc:
+        report["pymupdf_error"] = str(exc)
+    try:
+        import paddle
+
+        report["paddle"] = getattr(paddle, "__version__", "unknown")
+        report["paddle_cuda_compiled"] = bool(paddle.device.is_compiled_with_cuda())
+        report["paddle_devices"] = int(paddle.device.cuda.device_count())
+    except Exception as exc:
+        report["paddle_error"] = str(exc)
+
+    torch_cuda = False
+    torch_mps = False
+    try:
+        import torch
+
+        torch_cuda = bool(torch.cuda.is_available())
+        # ROCm-built torch reports itself through this SAME torch.cuda API
+        # (confusingly named) - True here does not necessarily mean NVIDIA.
+        mps_backend = getattr(torch.backends, "mps", None)
+        torch_mps = bool(mps_backend and mps_backend.is_available())
+    except Exception:
+        pass
+    report["torch_cuda"] = torch_cuda
+    report["torch_mps"] = torch_mps
+
+    # cuda=False (paddle has no usable NVIDIA device) previously looked
+    # IDENTICAL whether the machine had no GPU at all, or had a real
+    # AMD/Apple GPU torch can see but paddle's mainline wheel cannot (no
+    # ROCm/Metal build) - confirmed 2026-09-24, no such distinction existed
+    # anywhere in this codebase. Surfacing it matters: the core OCR
+    # pipeline (paddle-only) is correctly CPU-bound either way, but opt-in
+    # torch/ultralytics-based specialists (e.g. table_detector.py's YOLO26)
+    # may still be able to use a GPU paddle can't.
+    report["gpu_unsupported_by_paddle"] = bool(not report["cuda"] and (torch_cuda or torch_mps))
+    if report["gpu_unsupported_by_paddle"]:
+        report["gpu_note"] = (
+            f"A GPU is visible to torch (cuda={torch_cuda}, mps={torch_mps}) but not "
+            "to paddle - mainline paddlepaddle has no AMD ROCm or Apple Metal build, "
+            "so the core OCR pipeline will run on CPU. Opt-in torch-based specialists "
+            "(e.g. YOLO26 table detection) may still be able to use this GPU."
+        )
+
+    # Per-machine cached verdicts (see machine_state.py) - what this
+    # machine has already learned about itself, and how many times a
+    # timeout-guarded engine call has fallen back this process. Operators
+    # can use this to see WHY a tier is silently skipped without digging
+    # through logs.
+    report["cached_engine_status"] = machine_state.load_state().get("engines", {})
+    report["timeout_events"] = machine_state.get_timeout_stats()
+    return report
+
+
+def doctor() -> dict[str, object]:
+    return diagnose()
+
+
+__all__ = ["BackendMismatch", "check_backend", "diagnose", "doctor"]