core-pdf 0.0.2__py3-none-any.whl

core_pdf/__init__.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+from __future__ import annotations
+
+from core_pdf.impl.engine.extraction.document import PdfDocument
+from core_pdf.impl.engine.extraction.page import PdfPage
+from core_pdf.impl.exceptions import (
+    PdfError,
+    PdfParseError,
+    PdfSourceError,
+    PdfUnsupportedError,
+)
+from core_pdf.impl.types import PageSelection
+
+__all__ = (
+    "PageSelection",
+    "PdfDocument",
+    "PdfError",
+    "PdfPage",
+    "PdfParseError",
+    "PdfSourceError",
+    "PdfUnsupportedError",
+)

core_pdf/__main__.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+from __future__ import annotations
+
+from core_pdf.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

core_pdf/cli.py

@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from core_pdf import PdfDocument
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="core-pdf",
+        description="Extract PDF content with core-pdf.",
+    )
+    parser.add_argument("pdf", type=Path, help="Path to the PDF file")
+    parser.add_argument(
+        "--mode",
+        choices=("text", "markdown"),
+        default="text",
+        help="Plain output format",
+    )
+    parser.add_argument(
+        "--plain",
+        action="store_true",
+        help=argparse.SUPPRESS,
+    )
+    return parser
+
+
+def extract_output(document: PdfDocument, output_format: str) -> str:
+    if output_format == "markdown":
+        return document.to_markdown()
+    return document.extract_text()
+
+
+def print_plain_output(output: str) -> None:
+    sys.stdout.write(output)
+    if output and not output.endswith("\n"):
+        sys.stdout.write("\n")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        document = PdfDocument(args.pdf)
+    except Exception as exc:
+        print(f"core-pdf: {exc}", file=sys.stderr)
+        return 1
+
+    try:
+        print_plain_output(extract_output(document, args.mode))
+        return 0
+    except Exception as exc:
+        print(f"core-pdf: {exc}", file=sys.stderr)
+        return 1
+    finally:
+        document.__exit__(None, None, None)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

core_pdf/impl/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+from __future__ import annotations

core_pdf/impl/engine/__init__.py

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Internal PDF engine packages.
+
+Spec-aligned packages live under ``engine.spec`` and are prefixed by their
+PDF 32000 section:
+
+- ``s_07_*``: syntax, objects, filters, security, document structure, content streams
+- ``s_08_graphics``: graphics state, matrices, colour, images
+- ``s_09_fonts``: text/font data structures and text decoding
+- ``s_14_structure``: marked content, logical structure, tagged PDF
+
+Root-level packages are shared or derived processing layers:
+``layout``, ``extraction``, ``rendering``, and ``compat``.
+"""
+
+__all__: tuple[str, ...] = ()

core_pdf/impl/engine/extraction/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Derived text, table, redaction, and layout extraction."""
+
+__all__: tuple[str, ...] = ()

core_pdf/impl/engine/extraction/cache.py

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+from __future__ import annotations
+
+from collections.abc import Hashable, MutableMapping
+from typing import TypeAlias, TypeVar
+
+ExtractionCacheKey: TypeAlias = str | tuple[Hashable, ...]
+ExtractionCacheMapping: TypeAlias = MutableMapping[ExtractionCacheKey, object]
+
+_T = TypeVar("_T")
+
+
+class ExtractionCache(dict[ExtractionCacheKey, object]):
+    def get_as(self, key: ExtractionCacheKey, expected_type: type[_T]) -> _T | None:
+        value = self.get(key)
+        return value if isinstance(value, expected_type) else None

core_pdf/impl/engine/extraction/common/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Shared extraction utilities."""

core_pdf/impl/engine/extraction/common/observation_resolver.py

@@ -0,0 +1,211 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass, replace
+
+from core_pdf.impl.engine.extraction.common import page_geometry
+from core_pdf.impl.engine.extraction.ocr.text_analysis import normalized_text_tokens
+
+
+@dataclass(frozen=True)
+class ObservationResolution:
+    action: str
+    reason: str
+    candidate: page_geometry.PageObservation
+    matched: page_geometry.PageObservation | None = None
+    geometry_score: float = 0.0
+    coverage_ratio: float = 0.0
+    text_overlap: float = 0.0
+    useful_new_tokens: int = 0
+
+
+@dataclass(frozen=True)
+class ResolvedTextLine:
+    text: str
+    observation: page_geometry.PageObservation
+    break_before: int = 1
+    contributing_observations: tuple[page_geometry.PageObservation, ...] = ()
+    resolution: ObservationResolution | None = None
+
+
+def resolve_observation_append(
+    candidate: page_geometry.PageObservation,
+    accepted: Iterable[page_geometry.PageObservation],
+    *,
+    existing_text: str = "",
+) -> ObservationResolution:
+    accepted_observations = tuple(
+        observation for observation in accepted if observation.bbox is not None
+    )
+    if candidate.bbox is None:
+        if observation_has_useful_new_text(candidate, existing_text):
+            return ObservationResolution("append", "no_geometry", candidate)
+        return ObservationResolution("skip", "duplicate_text", candidate)
+    if not candidate.text.strip():
+        return ObservationResolution("skip", "empty_text", candidate)
+
+    matched, geometry_score = best_observation_geometry_match(
+        candidate,
+        accepted_observations,
+    )
+    text_overlap = observation_text_overlap(candidate, matched) if matched is not None else 0.0
+    coverage_ratio = observation_coverage_ratio(candidate, accepted_observations)
+    useful_new_tokens = observation_useful_new_token_count(candidate, existing_text)
+
+    if coverage_ratio >= 0.72 and text_overlap >= 0.65:
+        return ObservationResolution(
+            "skip",
+            "covered_geometry",
+            candidate,
+            matched,
+            geometry_score,
+            coverage_ratio,
+            text_overlap,
+            useful_new_tokens,
+        )
+    if geometry_score >= 0.86 and text_overlap >= 0.65:
+        return ObservationResolution(
+            "skip",
+            "same_geometry",
+            candidate,
+            matched,
+            geometry_score,
+            coverage_ratio,
+            text_overlap,
+            useful_new_tokens,
+        )
+    if geometry_score >= 0.70 and text_overlap >= 0.45:
+        return ObservationResolution(
+            "skip",
+            "same_text_region",
+            candidate,
+            matched,
+            geometry_score,
+            coverage_ratio,
+            text_overlap,
+            useful_new_tokens,
+        )
+    return ObservationResolution(
+        "append",
+        "uncovered",
+        candidate,
+        matched,
+        geometry_score,
+        coverage_ratio,
+        text_overlap,
+        useful_new_tokens,
+    )
+
+
+def resolve_text_lines(
+    lines: Iterable[ResolvedTextLine],
+    *,
+    existing_text: str = "",
+) -> tuple[ResolvedTextLine, ...]:
+    accepted_lines: list[ResolvedTextLine] = []
+    accepted_observations: list[page_geometry.PageObservation] = []
+    text_parts: list[str] = [existing_text] if existing_text else []
+    for line in lines:
+        observation = line.observation
+        if observation.text != line.text:
+            observation = replace(observation, text=line.text)
+            line = replace(line, observation=observation)
+        resolution = resolve_observation_append(
+            observation,
+            accepted_observations,
+            existing_text="\n".join(text_parts),
+        )
+        if resolution.action != "append":
+            continue
+        accepted_line = replace(line, resolution=resolution)
+        accepted_lines.append(accepted_line)
+        accepted_observations.append(observation)
+        text_parts.append(line.text)
+    return tuple(accepted_lines)
+
+
+def text_from_resolved_lines(lines: Iterable[ResolvedTextLine]) -> str:
+    parts: list[str] = []
+    for line in lines:
+        text = line.text.strip()
+        if not text:
+            continue
+        if parts:
+            parts.append("\n" * max(1, line.break_before))
+        parts.append(text)
+    return "".join(parts)
+
+
+def best_observation_geometry_match(
+    candidate: page_geometry.PageObservation,
+    accepted: Iterable[page_geometry.PageObservation],
+) -> tuple[page_geometry.PageObservation | None, float]:
+    best: page_geometry.PageObservation | None = None
+    best_score = 0.0
+    for observation in accepted:
+        score = page_geometry.observation_geometry_match_score(
+            candidate,
+            observation,
+        )
+        if score > best_score:
+            best = observation
+            best_score = score
+    return best, best_score
+
+
+def observation_coverage_ratio(
+    candidate: page_geometry.PageObservation,
+    accepted: Iterable[page_geometry.PageObservation],
+) -> float:
+    area = page_geometry.observation_area(candidate)
+    if area <= 0.0:
+        return 1.0
+    covered_area = 0.0
+    for observation in accepted:
+        covered_area += page_geometry.observation_intersection_area(
+            candidate,
+            observation,
+        )
+    return min(1.0, covered_area / area)
+
+
+def observation_text_overlap(
+    left: page_geometry.PageObservation,
+    right: page_geometry.PageObservation | None,
+) -> float:
+    if right is None:
+        return 0.0
+    left_tokens = set(normalized_text_tokens(left.text))
+    right_tokens = set(normalized_text_tokens(right.text))
+    if not left_tokens or not right_tokens:
+        return 0.0
+    return len(left_tokens & right_tokens) / max(1, min(len(left_tokens), len(right_tokens)))
+
+
+def observation_has_useful_new_text(
+    candidate: page_geometry.PageObservation,
+    existing_text: str,
+) -> bool:
+    return observation_useful_new_token_count(candidate, existing_text) > 0
+
+
+def observation_useful_new_token_count(
+    candidate: page_geometry.PageObservation,
+    existing_text: str,
+) -> int:
+    seen = set(normalized_text_tokens(existing_text))
+    count = 0
+    for token in normalized_text_tokens(candidate.text):
+        if token in seen:
+            continue
+        if token_has_content_signal(token):
+            count += 1
+    return count
+
+
+def token_has_content_signal(token: str) -> bool:
+    alnum = "".join(ch for ch in token if ch.isalnum())
+    if any(ch.isdigit() for ch in alnum):
+        return True
+    return len(alnum) >= 3