llmPDF 0.7.0__py3-none-any.whl

llmpdf/__init__.py

@@ -0,0 +1,36 @@
+"""Task-oriented llmPDF conversion pipeline."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("llmPDF")
+except PackageNotFoundError:  # Running directly from an unpacked source tree.
+    __version__ = "0+unknown"
+
+from .sdk import (
+    BillingSummary,
+    ConfigurationError,
+    ConversionError,
+    ConversionResult,
+    ConvertOptions,
+    DocumentConverterProtocol,
+    TaskExecutionError,
+    TokenUsage,
+    UsageSummary,
+    convert,
+)
+from .rerun import rerun_tables
+
+__all__ = [
+    "BillingSummary",
+    "ConfigurationError",
+    "ConversionError",
+    "ConversionResult",
+    "ConvertOptions",
+    "DocumentConverterProtocol",
+    "TaskExecutionError",
+    "TokenUsage",
+    "UsageSummary",
+    "convert",
+    "rerun_tables",
+]

llmpdf/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

llmpdf/agent_scheduler.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+import threading
+import time
+from collections import deque
+from collections.abc import Callable
+from concurrent.futures import Executor, Future
+from dataclasses import dataclass
+from typing import Any
+
+
+TASK_ORDER = ("find", "cross_table", "table", "image")
+
+
+@dataclass
+class _QueuedTask:
+    future: Future
+    kind: str
+    label: str
+    callback: Callable[..., Any]
+    args: tuple[Any, ...]
+    kwargs: dict[str, Any]
+    submitted_at: float
+    sequence: int
+
+
+class PriorityAgentExecutor(Executor):
+    """Fixed worker pool backed by ordered, per-task-type FIFO queues."""
+
+    def __init__(
+        self,
+        max_workers: int = 5,
+        *,
+        kind_limits: dict[str, int] | None = None,
+        progress: Callable[[str], None] | None = None,
+    ) -> None:
+        if not 1 <= max_workers <= 5:
+            raise ValueError("max_workers must be between 1 and 5")
+        self.max_workers = max_workers
+        configured_limits = kind_limits or {}
+        self._kind_limits = {
+            "find": max(1, int(configured_limits.get("find", max_workers))),
+            "table": max(1, int(configured_limits.get("table", max_workers))),
+            "image": max(1, int(configured_limits.get("image", max_workers))),
+        }
+        self._progress = progress
+        self._queues: dict[str, deque[_QueuedTask]] = {
+            kind: deque() for kind in TASK_ORDER
+        }
+        self._condition = threading.Condition()
+        self._shutdown = False
+        self._sequence = 0
+        self._running: dict[int, _QueuedTask] = {}
+        self._peak_running = 0
+        self._completed = 0
+        self._records: list[dict[str, Any]] = []
+        self._threads = [
+            threading.Thread(
+                target=self._worker,
+                name=f"pdf-agent-{index + 1}",
+                daemon=True,
+            )
+            for index in range(max_workers)
+        ]
+        for thread in self._threads:
+            thread.start()
+        self._emit(f"queue started: workers={max_workers}")
+
+    def submit(self, fn, /, *args, **kwargs):  # type: ignore[override]
+        return self.submit_task("table", getattr(fn, "__name__", "table"), fn, *args, **kwargs)
+
+    def submit_task(
+        self,
+        kind: str,
+        label: str,
+        fn: Callable[..., Any],
+        /,
+        *args: Any,
+        **kwargs: Any,
+    ) -> Future:
+        if kind not in self._queues:
+            raise ValueError(f"Unknown Agent task type: {kind}")
+        future: Future = Future()
+        with self._condition:
+            if self._shutdown:
+                raise RuntimeError("cannot schedule new futures after shutdown")
+            self._sequence += 1
+            task = _QueuedTask(
+                future=future,
+                kind=kind,
+                label=label,
+                callback=fn,
+                args=args,
+                kwargs=kwargs,
+                submitted_at=time.monotonic(),
+                sequence=self._sequence,
+            )
+            self._queues[kind].append(task)
+            state = self._state_locked()
+            self._condition.notify()
+        self._emit(f"queued {kind} {label}; {state}")
+        return future
+
+    def _running_count_locked(self, kind: str) -> int:
+        if kind in {"cross_table", "table"}:
+            return sum(
+                task.kind in {"cross_table", "table"}
+                for task in self._running.values()
+            )
+        return sum(task.kind == kind for task in self._running.values())
+
+    def _can_run_locked(self, kind: str) -> bool:
+        limit_kind = "table" if kind == "cross_table" else kind
+        return self._running_count_locked(kind) < self._kind_limits[limit_kind]
+
+    def _next_locked(self) -> _QueuedTask | None:
+        for kind in TASK_ORDER:
+            if self._queues[kind] and self._can_run_locked(kind):
+                return self._queues[kind].popleft()
+        return None
+
+    def _has_queued_locked(self) -> bool:
+        return any(self._queues[kind] for kind in TASK_ORDER)
+
+    def _has_runnable_locked(self) -> bool:
+        return any(
+            self._queues[kind] and self._can_run_locked(kind)
+            for kind in TASK_ORDER
+        )
+
+    def _state_locked(self) -> str:
+        ready = " ".join(
+            f"{kind}={len(self._queues[kind])}" for kind in TASK_ORDER
+        )
+        discovered = self._completed + len(self._running) + sum(
+            len(queue) for queue in self._queues.values()
+        )
+        return (
+            f"running={len(self._running)}/{self.max_workers} "
+            f"ready[{ready}] completed={self._completed} discovered={discovered}"
+        )
+
+    def _emit(self, message: str) -> None:
+        if self._progress is not None:
+            self._progress(f"[scheduler] {message}")
+
+    def _worker(self) -> None:
+        while True:
+            with self._condition:
+                self._condition.wait_for(
+                    lambda: self._has_runnable_locked()
+                    or (self._shutdown and not self._has_queued_locked())
+                )
+                task = self._next_locked()
+                if task is None:
+                    if self._shutdown and not self._has_queued_locked():
+                        return
+                    continue
+                if not task.future.set_running_or_notify_cancel():
+                    continue
+                worker_id = threading.get_ident()
+                self._running[worker_id] = task
+                self._peak_running = max(self._peak_running, len(self._running))
+                state = self._state_locked()
+            self._emit(f"started {task.kind} {task.label}; {state}")
+            started_at = time.monotonic()
+            try:
+                result = task.callback(*task.args, **task.kwargs)
+            except BaseException as error:
+                task.future.set_exception(error)
+                status = "failed"
+            else:
+                task.future.set_result(result)
+                status = "completed"
+            finished_at = time.monotonic()
+            with self._condition:
+                self._running.pop(worker_id, None)
+                self._completed += 1
+                self._records.append(
+                    {
+                        "sequence": task.sequence,
+                        "type": task.kind,
+                        "label": task.label,
+                        "status": status,
+                        "queue_wait_seconds": round(started_at - task.submitted_at, 3),
+                        "run_seconds": round(finished_at - started_at, 3),
+                    }
+                )
+                state = self._state_locked()
+                self._condition.notify_all()
+            self._emit(
+                f"{status} {task.kind} {task.label} in "
+                f"{finished_at - started_at:.1f}s; {state}"
+            )
+
+    def snapshot(self) -> dict[str, Any]:
+        with self._condition:
+            return {
+                "task_order": list(TASK_ORDER),
+                "max_workers": self.max_workers,
+                "kind_limits": dict(self._kind_limits),
+                "completed": self._completed,
+                "running": len(self._running),
+                "peak_running": self._peak_running,
+                "ready": {
+                    kind: len(self._queues[kind]) for kind in TASK_ORDER
+                },
+                "tasks": list(self._records),
+            }
+
+    def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:
+        with self._condition:
+            self._shutdown = True
+            if cancel_futures:
+                for queue in self._queues.values():
+                    while queue:
+                        queue.popleft().future.cancel()
+            self._condition.notify_all()
+        if wait:
+            for thread in self._threads:
+                thread.join()

llmpdf/assets_task.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import csv
+import re
+import shutil
+from pathlib import Path
+
+import yaml
+
+from .io_utils import copy_file, read_json, relativize, sha256_file, write_json
+from .models import BBox, PipelineConfig, TaskResult
+from .task import PipelineTask
+
+
+def infer_header_rows(name: str | None) -> int:
+    normalized = (name or "").casefold()
+    headerless_terms = ("abkürz", "abbreviation", "glossar", "glossary")
+    return 0 if any(term in normalized for term in headerless_terms) else 1
+
+
+def csv_to_markdown(
+    csv_path: Path, title: str | None, header_rows: int | None = None
+) -> str:
+    with csv_path.open(encoding="utf-8-sig", newline="") as stream:
+        rows = list(csv.reader(stream))
+    if not rows:
+        return ""
+    width = max(len(row) for row in rows)
+    rows = [row + [""] * (width - len(row)) for row in rows]
+
+    def clean(value: str) -> str:
+        return re.sub(r"\s*[\r\n]+\s*", " ", value).replace("|", "\\|")
+
+    lines = []
+    if title:
+        lines.extend([f"**{title.strip()}**", ""])
+    header_rows = infer_header_rows(title) if header_rows is None else header_rows
+    header = rows[0] if header_rows else [""] * width
+    body = rows[1:] if header_rows else rows
+    lines.append("| " + " | ".join(clean(value) for value in header) + " |")
+    lines.append("| " + " | ".join("---" for _ in range(width)) + " |")
+    lines.extend(
+        "| " + " | ".join(clean(value) for value in row) + " |" for row in body
+    )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+class CollectAssetsTask(PipelineTask):
+    name = "06-collect-assets"
+    dependencies = ("05-extract-tables",)
+
+    def signature_payload(self, config: PipelineConfig) -> dict:
+        value = super().signature_payload(config)
+        value["asset_logic_version"] = 6
+        summary = config.work_dir / "table-extraction" / "summary.json"
+        if summary.is_file():
+            value["extraction_summary_sha256"] = sha256_file(summary)
+            extraction = read_json(summary)
+            artifacts: dict[str, str] = {}
+            for relative_directory in extraction.get("table_directories", []):
+                directory = config.output_dir / relative_directory
+                for pattern in ("output_*.csv", "extract.py", "metadata.yaml"):
+                    for path in sorted(directory.glob(pattern)):
+                        artifacts[relativize(path, config.output_dir)] = sha256_file(
+                            path
+                        )
+            value["extraction_artifacts"] = artifacts
+        return value
+
+    def run(self, config: PipelineConfig) -> TaskResult:
+        extraction = read_json(config.work_dir / "table-extraction" / "summary.json")
+        source_dirs = [
+            config.output_dir / path for path in extraction.get("table_directories", [])
+        ]
+        continuation_by_leader = {
+            int(group["leader_page"]): [int(page) for page in group["pages"]]
+            for group in extraction.get("continuation_groups", [])
+        }
+        records = []
+        sortable = []
+        for source_dir in source_dirs:
+            metadata = yaml.safe_load(
+                (source_dir / "metadata.yaml").read_text(encoding="utf-8")
+            )
+            sortable.append(
+                (
+                    int(metadata["page"]),
+                    int(metadata["page_table_index"]),
+                    source_dir,
+                    metadata,
+                )
+            )
+        sortable.sort()
+
+        internal_root = config.work_dir / "table-assets"
+        tables_root = config.assets_dir / "tables"
+        if config.force:
+            for path in (internal_root, tables_root):
+                if path.exists():
+                    shutil.rmtree(path)
+        internal_root.mkdir(parents=True, exist_ok=True)
+        tables_root.mkdir(parents=True, exist_ok=True)
+        for global_index, (page, page_index, source_dir, metadata) in enumerate(
+            sortable, 1
+        ):
+            table_id = f"table-{global_index:04d}"
+            internal = internal_root / table_id
+            internal.mkdir(parents=True, exist_ok=True)
+            csv_paths = sorted(source_dir.glob("output_*.csv"))
+            if not csv_paths:
+                continue
+            primary_csv = copy_file(csv_paths[0], tables_root / f"{table_id}.csv")
+            extra_csvs = []
+            for extra_index, extra in enumerate(csv_paths[1:], 2):
+                extra_csvs.append(
+                    copy_file(extra, tables_root / f"{table_id}-{extra_index}.csv")
+                )
+            extractor = copy_file(source_dir / "extract.py", internal / "extract.py")
+            source_metadata = copy_file(
+                source_dir / "metadata.yaml", internal / "metadata.yaml"
+            )
+            markdown_path = internal / "table.md"
+            header_rows = infer_header_rows(metadata.get("name"))
+            markdown_path.write_text(
+                csv_to_markdown(primary_csv, metadata.get("name"), header_rows),
+                encoding="utf-8",
+            )
+
+            bbox_data = metadata["bbox"]
+            bbox = BBox.from_dict(bbox_data)
+            source_pages = [
+                int(value) for value in metadata.get("source_pages", [page])
+            ]
+            if source_pages == [page] and page in continuation_by_leader:
+                anchored = [
+                    item
+                    for item in sortable
+                    if int(item[0]) == page and f"page-{page:04d}" in item[2].parts
+                ]
+                # Older Agent outputs did not record source_pages. The fallback
+                # is safe only when the continuation leader has one anchored table.
+                if len(anchored) == 1:
+                    source_pages = continuation_by_leader[page]
+            record = {
+                "id": table_id,
+                "page": page,
+                "source_pages": source_pages,
+                "page_table_index": page_index,
+                "name": metadata.get("name"),
+                "header_rows": header_rows,
+                "bbox": {
+                    "coordinate_system": "pdfplumber_top_left",
+                    "unit": "pt",
+                    **bbox.to_dict(),
+                },
+                "csv": relativize(primary_csv, config.output_dir),
+                "extra_csvs": [
+                    relativize(path, config.output_dir) for path in extra_csvs
+                ],
+                "internal": {
+                    "markdown": relativize(markdown_path, config.output_dir),
+                    "extractor": relativize(extractor, config.output_dir),
+                    "metadata": relativize(source_metadata, config.output_dir),
+                },
+            }
+            records.append(record)
+        manifest = internal_root / "tables.json"
+        write_json(manifest, {"schema_version": 2, "tables": records})
+        return TaskResult(
+            self.name,
+            "completed",
+            [relativize(manifest, config.output_dir)],
+            {"table_count": len(records)},
+        )