tracecite 0.0.2__py3-none-any.whl

tracecite/evidence/manifest.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+import fnmatch
+import tomllib
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import MappingProxyType
+
+from .paths import PathAuthorityError, normalise_relative_declaration
+
+SOURCE_TYPES = {
+    ".pdf": "pdf",
+    ".md": "markdown",
+    ".xlsx": "workbook",
+    ".xlsm": "workbook",
+}
+_TOP_LEVEL_KEYS = {
+    "schema_version",
+    "root",
+    "database",
+    "model_cache_dir",
+    "source",
+    "include",
+    "exclude",
+}
+_TABLE_KEYS = {
+    "source": {"path", "origin"},
+    "include": {"glob", "origin"},
+    "exclude": {"glob", "origin"},
+}
+_REQUIRED_TABLE_KEYS = {
+    "source": {"path"},
+    "include": {"glob"},
+    "exclude": {"glob"},
+}
+_ORIGINS = {"default", "local"}
+
+
+class ManifestError(ValueError):
+    pass
+
+
+def _normalise(value: str | Path) -> str:
+    try:
+        normalised = normalise_relative_declaration(value)
+    except PathAuthorityError as exc:
+        raise ManifestError(str(exc)) from exc
+    if Path(normalised).suffix.lower() not in SOURCE_TYPES and not any(
+        ch in normalised for ch in "*?["
+    ):
+        raise ManifestError(f"unsupported source extension for {value}")
+    return normalised
+
+
+def _matches(path: str, pattern: str) -> bool:
+    return fnmatch.fnmatchcase(path, pattern) or (
+        "**/" in pattern and fnmatch.fnmatchcase(path, pattern.replace("**/", ""))
+    )
+
+
+@dataclass(frozen=True)
+class ManifestRules:
+    explicit_paths: tuple[str, ...] = ()
+    include_globs: tuple[str, ...] = ()
+    exclude_globs: tuple[str, ...] = ()
+    origins: Mapping[tuple[str, str], str] = field(
+        default_factory=lambda: MappingProxyType({})
+    )
+
+    def selects_path(self, path: str) -> bool:
+        path = str(path).replace("\\", "/")
+        selected = path in self.explicit_paths or any(
+            _matches(path, glob) for glob in self.include_globs
+        )
+        excluded = any(_matches(path, glob) for glob in self.exclude_globs)
+        return selected and not excluded
+
+
+@dataclass(frozen=True)
+class SourceSelection:
+    selected: dict[str, Path]
+    missing_explicit_paths: tuple[str, ...]
+    unmatched_globs: tuple[str, ...]
+
+
+def load_manifest(path: Path) -> ManifestRules:
+    with Path(path).open("rb") as handle:
+        data = tomllib.load(handle)
+    if data.get("schema_version") != 1:
+        raise ManifestError(f"{path} requires schema_version = 1")
+    unknown = set(data) - _TOP_LEVEL_KEYS
+    if unknown:
+        raise ManifestError(f"unknown field(s) in {path}: {', '.join(sorted(unknown))}")
+
+    origins: dict[tuple[str, str], str] = {}
+
+    def table_values(name: str, field: str) -> tuple[str, ...]:
+        values = []
+        for index, item in enumerate(data.get(name, []), start=1):
+            if not isinstance(item, dict):
+                raise ManifestError(f"[[{name}]] entry {index} must be a table")
+            keys = set(item)
+            unknown_keys = keys - _TABLE_KEYS[name]
+            missing = _REQUIRED_TABLE_KEYS[name] - keys
+            if unknown_keys:
+                raise ManifestError(
+                    f"unknown field(s) in [[{name}]] entry {index}: {', '.join(sorted(unknown_keys))}"
+                )
+            if missing:
+                raise ManifestError(
+                    f"missing field(s) in [[{name}]] entry {index}: {', '.join(sorted(missing))}"
+                )
+            origin = item.get("origin", "default")
+            if origin not in _ORIGINS:
+                raise ManifestError(
+                    f"unsupported origin in [[{name}]] entry {index}: {origin}"
+                )
+            identity = _normalise(item[field])
+            values.append(identity)
+            key = (name, identity)
+            if origins.get(key) != "local" or origin == "local":
+                origins[key] = origin
+        return tuple(values)
+
+    explicit = table_values("source", "path")
+    includes = table_values("include", "glob")
+    excludes = table_values("exclude", "glob")
+    return ManifestRules(
+        tuple(dict.fromkeys(explicit)),
+        tuple(dict.fromkeys(includes)),
+        tuple(dict.fromkeys(excludes)),
+        MappingProxyType(origins),
+    )
+
+
+def merge_manifests(paths: Sequence[Path]) -> ManifestRules:
+    """Union manifests in order; a ``local`` origin wins over ``default`` for the same declaration."""
+    declared: dict[str, dict[str, None]] = {"source": {}, "include": {}, "exclude": {}}
+    origins: dict[tuple[str, str], str] = {}
+    for path in paths:
+        rules = load_manifest(path)
+        for kind, values in (
+            ("source", rules.explicit_paths),
+            ("include", rules.include_globs),
+            ("exclude", rules.exclude_globs),
+        ):
+            for value in values:
+                declared[kind][value] = None
+                origin = rules.origins.get((kind, value), "default")
+                if origins.get((kind, value)) != "local" or origin == "local":
+                    origins[(kind, value)] = origin
+    return ManifestRules(
+        tuple(declared["source"]),
+        tuple(declared["include"]),
+        tuple(declared["exclude"]),
+        MappingProxyType(origins),
+    )
+
+
+def resolve_sources(root: Path, rules: ManifestRules) -> SourceSelection:
+    """Scan the declared paths and globs once under ``root``.
+
+    ``selected`` maps each existing, non-symlink, supported file that the
+    rules select to its resolved path. Missing explicit paths and include
+    globs that match no file are judged before exclusions apply.
+    """
+    root = Path(root).resolve()
+    candidates = {root / path for path in rules.explicit_paths}
+    for pattern in rules.include_globs:
+        candidates.update(root.glob(pattern))
+    declared: dict[str, Path] = {}
+    for path in sorted(candidates):
+        if (
+            path.is_symlink()
+            or not path.is_file()
+            or path.suffix.lower() not in SOURCE_TYPES
+        ):
+            continue
+        resolved = path.resolve()
+        if resolved.is_relative_to(root):
+            declared[resolved.relative_to(root).as_posix()] = resolved
+    return SourceSelection(
+        selected={
+            path: resolved
+            for path, resolved in declared.items()
+            if rules.selects_path(path)
+        },
+        missing_explicit_paths=tuple(
+            path for path in rules.explicit_paths if path not in declared
+        ),
+        unmatched_globs=tuple(
+            glob
+            for glob in rules.include_globs
+            if not any(_matches(path, glob) for path in declared)
+        ),
+    )

tracecite/evidence/page_extraction.py

@@ -0,0 +1,200 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from .page_selection import (
+    PageSelectionSyntaxError,
+    PageSelectionUnavailableError,
+    resolve_page_selection,
+)
+from .paths import (
+    PathAuthorityError,
+    file_sha256,
+    normalise_source_path,
+    source_row_for_path,
+)
+
+
+class PageExtractionError(ValueError):
+    pass
+
+
+class PageExtractionMissingError(PageExtractionError):
+    """The source or a selected page is not indexed or not on disk."""
+
+
+@dataclass(frozen=True)
+class PageExtractionResult:
+    pdf_path: Path
+    manifest_path: Path
+    normalized_pages: list[int]
+
+
+def _slugify(text: str, *, limit: int = 24) -> str:
+    slug = re.sub(r"[^A-Za-z0-9._-]+", "-", text).strip(".-_")
+    return (slug or "source")[:limit]
+
+
+def _selector_token(selector: str | None, normalized_pages: list[int]) -> str:
+    raw = selector if selector is not None else "1"
+    return hashlib.sha256(
+        f"{raw}|{','.join(map(str, normalized_pages))}".encode()
+    ).hexdigest()[:12]
+
+
+def _bounded_name(
+    source_stem: str, selector: str | None, normalized_pages: list[int], suffix: str
+) -> str:
+    base = f"tracecite-extract-{_slugify(source_stem)}-{_selector_token(selector, normalized_pages)}"
+    return f"{base[:72]}{suffix}"
+
+
+def _ensure_real_output_dir(output_dir: Path) -> Path:
+    output_dir = Path(output_dir)
+    if not output_dir.exists() or not output_dir.is_dir():
+        raise PageExtractionError(f"output directory does not exist: {output_dir}")
+    if output_dir.is_symlink():
+        raise PageExtractionError(
+            f"output directory must not be a symlink: {output_dir}"
+        )
+    return output_dir.resolve()
+
+
+def promote_staged_outputs(
+    staged_pdf: Path, staged_manifest: Path, final_pdf: Path, final_manifest: Path
+) -> None:
+    reserved: list[Path] = []
+    try:
+        for final_path in (final_pdf, final_manifest):
+            try:
+                descriptor = os.open(
+                    final_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600
+                )
+            except FileExistsError as exc:
+                raise PageExtractionError(
+                    f"output already exists: {final_path}"
+                ) from exc
+            else:
+                os.close(descriptor)
+                reserved.append(final_path)
+        os.replace(staged_pdf, final_pdf)
+        os.replace(staged_manifest, final_manifest)
+    except Exception:
+        for final_path in reserved:
+            final_path.unlink(missing_ok=True)
+        raise
+
+
+def extract_pages(
+    conn,
+    root: Path,
+    source_path_input: str,
+    selector: str | None,
+    output_dir: Path,
+) -> PageExtractionResult:
+    try:
+        source_path = normalise_source_path(root, source_path_input)
+    except PathAuthorityError as exc:
+        raise PageExtractionError(str(exc)) from exc
+
+    source_row = source_row_for_path(conn, source_path)
+    if source_row is None:
+        raise PageExtractionMissingError(f"source not found: {source_path}")
+    if source_row["source_type"] != "pdf":
+        raise PageExtractionError(f"source is not a pdf: {source_path}")
+
+    available_pages = [
+        row["physical_page"]
+        for row in conn.execute(
+            "SELECT physical_page FROM pages WHERE source_pk = ? ORDER BY physical_page",
+            (source_row["source_pk"],),
+        ).fetchall()
+    ]
+    if not available_pages:
+        raise PageExtractionMissingError(f"no indexed pages for source: {source_path}")
+
+    try:
+        normalized_pages = resolve_page_selection(selector, available_pages)
+    except PageSelectionSyntaxError as exc:
+        raise PageExtractionError(str(exc)) from exc
+    except PageSelectionUnavailableError as exc:
+        raise PageExtractionMissingError(
+            f"page {exc.page} not indexed for source: {source_path}"
+        ) from exc
+
+    output_dir_resolved = _ensure_real_output_dir(output_dir)
+    root_resolved = root.resolve()
+    if (
+        output_dir_resolved == root_resolved
+        or output_dir_resolved.is_relative_to(root_resolved)
+        or root_resolved.is_relative_to(output_dir_resolved)
+    ):
+        raise PageExtractionError(
+            f"output directory overlaps source root: {output_dir}"
+        )
+    source_pdf = root / source_path
+    if not source_pdf.is_file():
+        raise PageExtractionMissingError(f"source pdf missing: {source_path}")
+    source_pdf_resolved = source_pdf.resolve()
+    source_sha256 = file_sha256(source_pdf_resolved)
+    if source_sha256 != source_row["sha256"]:
+        raise PageExtractionError(f"source pdf changed since indexing: {source_path}")
+
+    import pymupdf
+
+    with pymupdf.open(source_pdf_resolved) as document:
+        if document.page_count < max(normalized_pages):
+            raise PageExtractionError(
+                f"source pdf has only {document.page_count} page(s), cannot satisfy selection {normalized_pages}"
+            )
+
+        pdf_name = _bounded_name(
+            Path(source_path).stem, selector, normalized_pages, ".pdf"
+        )
+        manifest_name = _bounded_name(
+            Path(source_path).stem, selector, normalized_pages, ".json"
+        )
+        final_pdf = output_dir_resolved / pdf_name
+        final_manifest = output_dir_resolved / manifest_name
+        if final_pdf.exists() or final_manifest.exists():
+            raise PageExtractionError(
+                f"output already exists: {final_pdf if final_pdf.exists() else final_manifest}"
+            )
+
+        with TemporaryDirectory(dir=output_dir_resolved.parent) as temp_root:
+            staged_pdf = Path(temp_root) / pdf_name
+            staged_manifest = Path(temp_root) / manifest_name
+            with pymupdf.open() as derivative:
+                for page in normalized_pages:
+                    derivative.insert_pdf(
+                        document, from_page=page - 1, to_page=page - 1
+                    )
+                derivative.save(staged_pdf)
+            manifest = {
+                "schema_version": 1,
+                "source_path": source_path,
+                "original_selector": selector,
+                "selector": selector if selector is not None else "1",
+                "normalized_pages": normalized_pages,
+                "page_count": len(normalized_pages),
+                "source_page_count": document.page_count,
+                "source_sha256": source_sha256,
+                "derivative_filename": pdf_name,
+                "derivative_sha256": file_sha256(staged_pdf),
+                "created_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
+            }
+            staged_manifest.write_text(
+                json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+            )
+            promote_staged_outputs(
+                staged_pdf, staged_manifest, final_pdf, final_manifest
+            )
+
+    return PageExtractionResult(final_pdf, final_manifest, normalized_pages)

tracecite/evidence/page_output.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from . import schema
+from .paths import file_sha256
+
+
+class PageOutputError(ValueError):
+    pass
+
+
+class PageAssetMissingError(PageOutputError):
+    pass
+
+
+def _asset_payload(database_path: Path, asset_row) -> dict:
+    try:
+        asset_path = schema.resolve_asset_path(database_path, asset_row["asset_path"])
+    except ValueError as exc:
+        raise PageOutputError(str(exc)) from exc
+    resolved_path = asset_path.resolve()
+    if not asset_path.is_file():
+        raise PageAssetMissingError(f"missing asset file: {asset_row['asset_path']}")
+    if file_sha256(asset_path) != asset_row["sha256"]:
+        raise PageAssetMissingError(f"asset hash mismatch: {asset_row['asset_path']}")
+    try:
+        bbox = json.loads(asset_row["bbox_json"]) if asset_row["bbox_json"] else None
+    except ValueError as exc:
+        raise PageOutputError(
+            f"invalid asset bbox json: {asset_row['asset_path']}"
+        ) from exc
+    return {
+        "asset_path": asset_row["asset_path"],
+        "resolved_path": str(resolved_path),
+        "asset_type": asset_row["asset_type"],
+        "sha256": asset_row["sha256"],
+        "width": asset_row["width"],
+        "height": asset_row["height"],
+        "label": asset_row["label"],
+        "caption": asset_row["caption"],
+        "nearby_text": asset_row["nearby_text"],
+        "bbox": bbox,
+    }
+
+
+def page_assets_for(
+    conn, database_path: Path, source_pk: int, physical_page: int
+) -> tuple[dict | None, list[dict]]:
+    rows = conn.execute(
+        "SELECT * FROM assets WHERE source_pk = ? AND physical_page = ? ORDER BY asset_type, asset_path",
+        (source_pk, physical_page),
+    ).fetchall()
+    page_render = None
+    figure_crops: list[dict] = []
+    for row in rows:
+        payload = _asset_payload(database_path, row)
+        if row["asset_type"] == "page-render" and page_render is None:
+            page_render = payload
+        elif row["asset_type"] == "figure-crop":
+            figure_crops.append(payload)
+    return page_render, figure_crops
+
+
+def page_json_payload(conn, database_path: Path, source_row, page_row) -> dict:
+    page_render, figure_crops = page_assets_for(
+        conn, database_path, source_row["source_pk"], page_row["physical_page"]
+    )
+    return {
+        "source_path": source_row["path"],
+        "physical_page": page_row["physical_page"],
+        "text": page_row["text"],
+        "printed_label": page_row["printed_label"],
+        "extraction_method": page_row["extraction_method"],
+        "extraction_status": page_row["extraction_status"],
+        "pdf_link": f"{source_row['path']}#page={page_row['physical_page']}",
+        "page_render": page_render,
+        "figure_crops": figure_crops,
+    }

tracecite/evidence/page_selection.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+
+class PageSelectionError(ValueError):
+    pass
+
+
+class PageSelectionSyntaxError(PageSelectionError):
+    pass
+
+
+class PageSelectionUnavailableError(PageSelectionError):
+    def __init__(self, page: int, message: str | None = None) -> None:
+        self.page = page
+        super().__init__(message or f"page {page} is unavailable")
+
+
+def _validate_positive_page(text: str, *, selector: str) -> int:
+    if not text.isdigit() or text.startswith("0"):
+        raise PageSelectionSyntaxError(f"invalid page selector: {selector}")
+    page = int(text)
+    if page <= 0:
+        raise PageSelectionSyntaxError(f"invalid page selector: {selector}")
+    return page
+
+
+def _expand_closed_range(
+    start: int, end: int, *, available_set: set[int], selector: str
+) -> list[int]:
+    if start > end:
+        raise PageSelectionSyntaxError(f"invalid page selector: {selector}")
+    pages = list(range(start, end + 1))
+    missing = next((page for page in pages if page not in available_set), None)
+    if missing is not None:
+        raise PageSelectionUnavailableError(missing)
+    return pages
+
+
+def resolve_page_selection(
+    selector: str | None,
+    available_pages: Sequence[int],
+    *,
+    default_page: int = 1,
+) -> list[int]:
+    available = sorted({int(page) for page in available_pages})
+    available_set = set(available)
+    if selector is None:
+        if default_page not in available_set:
+            raise PageSelectionUnavailableError(default_page)
+        return [default_page]
+
+    selector = selector.strip()
+    if not selector:
+        raise PageSelectionSyntaxError("invalid page selector: empty selector")
+
+    terms = [term.strip() for term in selector.split(",")]
+    if any(term == "" for term in terms):
+        raise PageSelectionSyntaxError(f"invalid page selector: {selector}")
+
+    if len(terms) == 1 and terms[0] == "all":
+        if not available:
+            raise PageSelectionUnavailableError(default_page)
+        return available
+    if any(term == "all" for term in terms):
+        raise PageSelectionSyntaxError(f"invalid page selector: {selector}")
+
+    resolved: set[int] = set()
+    max_available = available[-1] if available else None
+    for term in terms:
+        if term.endswith("-") and term != "-":
+            start = _validate_positive_page(term[:-1], selector=selector)
+            if not available:
+                raise PageSelectionUnavailableError(default_page)
+            assert max_available is not None
+            if start > max_available:
+                raise PageSelectionUnavailableError(start)
+            resolved.update(
+                _expand_closed_range(
+                    start, max_available, available_set=available_set, selector=selector
+                )
+            )
+            continue
+        if term.startswith("-") and term != "-":
+            end = _validate_positive_page(term[1:], selector=selector)
+            if not available:
+                raise PageSelectionUnavailableError(default_page)
+            resolved.update(
+                _expand_closed_range(
+                    1, end, available_set=available_set, selector=selector
+                )
+            )
+            continue
+        if "-" in term:
+            start_text, end_text = term.split("-", 1)
+            if not start_text or not end_text:
+                raise PageSelectionSyntaxError(f"invalid page selector: {selector}")
+            start = _validate_positive_page(start_text, selector=selector)
+            end = _validate_positive_page(end_text, selector=selector)
+            if not available:
+                raise PageSelectionUnavailableError(default_page)
+            resolved.update(
+                _expand_closed_range(
+                    start, end, available_set=available_set, selector=selector
+                )
+            )
+            continue
+        page = _validate_positive_page(term, selector=selector)
+        if not available:
+            raise PageSelectionUnavailableError(default_page)
+        if page not in available_set:
+            raise PageSelectionUnavailableError(page)
+        resolved.add(page)
+
+    return sorted(resolved)

tracecite/evidence/parsers/__init__.py

@@ -0,0 +1 @@
+"""Format-specific parsers behind a common interface (see ``base.py``)."""

tracecite/evidence/parsers/base.py

@@ -0,0 +1,65 @@
+"""Parser interface: authoritative files in, retained pages + logical chunk
+input units out. Chunking (chunking.py) turns ``ParsedChunkUnit`` sequences
+into stored chunks; parsers never write to the database directly.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class ParsedPage:
+    """One retained extraction container, such as a page or worksheet."""
+
+    physical_page: int
+    printed_label: str | None
+    text: str
+    extraction_method: str
+    extraction_status: str
+    section_candidates: list[str] = field(default_factory=list)
+    layout: dict | None = None
+
+
+@dataclass(frozen=True)
+class ParsedChunkUnit:
+    """One chunkable unit of text with enough locator context to build a
+    stable logical key and, later, a semantic-input hash.
+    """
+
+    text: str
+    logical_key: str
+    heading_path: list[str]
+    symbol: str | None
+    content_type: str
+    physical_page: int | None = None
+    page_start_offset: int | None = None
+    page_end_offset: int | None = None
+    line_start: int | None = None
+    line_end: int | None = None
+    locator: dict = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class ParseResult:
+    pages: list[ParsedPage]
+    units: list[ParsedChunkUnit]
+
+
+def units_from_layout(layout_json: str | None) -> list[ParsedChunkUnit]:
+    """Rebuild Markdown or workbook chunk-input units from retained ``pages.layout_json``."""
+    layout = json.loads(layout_json) if layout_json else {}
+    return [
+        ParsedChunkUnit(
+            text=entry["text"],
+            logical_key=entry["logical_key"],
+            heading_path=entry["heading_path"],
+            symbol=None,
+            content_type=entry["content_type"],
+            line_start=entry.get("line_start"),
+            line_end=entry.get("line_end"),
+            locator=entry.get("locator", {}),
+        )
+        for entry in layout.get("units", [])
+    ]