paperstack-cli 0.3.2__py3-none-any.whl → 0.4.0__py3-none-any.whl

paperstack/cli.py

@@ -1004,10 +1004,13 @@ def _run_paper(a: argparse.Namespace) -> int:
         from .content import arxiv_pdf
 
         arxiv_pdf.CACHE_DIR = _paper_cache()
-        cached_pdf = arxiv_pdf.CACHE_DIR / ref.value / "paper.md"
-        if offline and (not cached_pdf.is_file() or cached_pdf.stat().st_size <= 1000):
-            die(f"no complete cached PDF conversion for arxiv:{ref.value}")
-        if not arxiv_pdf.convert(ref.value):
+        cached_pdf = arxiv_pdf._cached_conversion(
+            arxiv_pdf.CACHE_DIR / ref.value,
+            allow_native_fallback=offline,
+        )
+        if offline and cached_pdf is None:
+            die(f"no usable cached PDF conversion for arxiv:{ref.value}")
+        if not arxiv_pdf.convert(ref.value, allow_native_fallback=offline):
             return 1
         return 0
     raise AssertionError(a.paper_cmd)

paperstack/content/arxiv_pdf.py

@@ -1,4 +1,4 @@
-"""Convert native-PDF arXiv submissions to Markdown.
+"""Convert arXiv PDF submissions to Markdown with selective OCR.
 
 Used by `paperstack paper pdf` after the source fetcher reports no TeX.
 """
@@ -6,9 +6,11 @@ Used by `paperstack paper pdf` after the source fetcher reports no TeX.
 from __future__ import annotations
 
 import hashlib
+import importlib.metadata
 import json
 import os
 import sys
+import tempfile
 import time
 import urllib.error
 import urllib.request
@@ -16,6 +18,9 @@ from pathlib import Path
 
 _CACHE_ROOT = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
 CACHE_DIR = Path(os.environ.get("PAPERSTACK_PAPERS_DIR", _CACHE_ROOT / "paperstack" / "papers"))
+CONVERTER = "pdf-inspector"
+MIN_MARKDOWN_CHARS = 100
+OCR_RUNTIME_GUIDE = "https://github.com/firecrawl/pdf-inspector/blob/main/docs/ocr-runtime.md"
 
 
 def _fetch(url: str, timeout: int = 60) -> bytes | None:
@@ -38,15 +43,139 @@ def _fetch(url: str, timeout: int = 60) -> bytes | None:
     return None
 
 
-def convert(arxiv_id: str) -> bool:
-    d = CACHE_DIR / arxiv_id
+def _cached_conversion(d: Path, *, allow_native_fallback: bool = False) -> Path | None:
     md_path = d / "paper.md"
-    if md_path.exists() and md_path.stat().st_size > 1000:
-        print(f"{arxiv_id}: cached at {md_path}")
+    meta_path = d / "meta.json"
+    if not md_path.is_file() or md_path.stat().st_size < MIN_MARKDOWN_CHARS or not meta_path.is_file():
+        return None
+    try:
+        markdown = md_path.read_bytes()
+        meta = json.loads(meta_path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+        return None
+    if not isinstance(meta, dict):
+        return None
+    valid = (
+        meta.get("converter") == CONVERTER
+        and meta.get("bytes") == len(markdown)
+        and meta.get("sha256") == hashlib.sha256(markdown).hexdigest()
+        and (allow_native_fallback or meta.get("conversion_mode") != "native_fallback")
+    )
+    return md_path if valid else None
+
+
+def _converter_version() -> str:
+    try:
+        return importlib.metadata.version(CONVERTER)
+    except importlib.metadata.PackageNotFoundError:
+        return "unknown"
+
+
+def _ocr_reasons(items) -> list[dict]:
+    return [{"page": item.page, "reasons": list(item.reasons)} for item in items]
+
+
+def _page_provenance(page) -> dict:
+    provenance = page.provenance
+    model = provenance.ocr_model
+    return {
+        "page": page.page_number,
+        "source": provenance.source,
+        "ocr_model": None if model is None else {"name": model.name, "revision": model.revision},
+        "render_dpi": provenance.render_dpi,
+        "ocr_confidence": provenance.ocr_confidence,
+        "hosted_recommended": provenance.hosted_recommended,
+        "warnings": list(provenance.warnings),
+        "timings": {
+            "render_ms": provenance.timings.render_ms,
+            "ocr_ms": provenance.timings.ocr_ms,
+            "assembly_ms": provenance.timings.assembly_ms,
+        },
+    }
+
+
+def _base_meta(arxiv_id: str, url: str, markdown: str) -> dict:
+    encoded = markdown.encode("utf-8")
+    return {
+        "schema_version": 1,
+        "arxiv_id": arxiv_id,
+        "bytes": len(encoded),
+        "sha256": hashlib.sha256(encoded).hexdigest(),
+        "url": url,
+        "converter": CONVERTER,
+        "converter_version": _converter_version(),
+    }
+
+
+def _ocr_meta(arxiv_id: str, url: str, markdown: str, result) -> dict:
+    meta = _base_meta(arxiv_id, url, markdown)
+    meta.update(
+        {
+            "conversion_mode": "selective_ocr",
+            "quality": "partial" if result.pages_recommending_hosted else "complete",
+            "page_count": result.page_count,
+            "pages_recommended_for_ocr": list(result.pages_recommended_for_ocr),
+            "pages_routed_to_ocr": list(result.pages_routed_to_ocr),
+            "pages_recommending_hosted": list(result.pages_recommending_hosted),
+            "ocr_reasons_by_page": _ocr_reasons(result.ocr_reasons_by_page),
+            "pages_with_tables": list(result.pages_with_tables),
+            "pages_with_columns": list(result.pages_with_columns),
+            "pages": [_page_provenance(page) for page in result.pages],
+        }
+    )
+    return meta
+
+
+def _native_fallback_meta(arxiv_id: str, url: str, markdown: str, result, error: ValueError) -> dict:
+    meta = _ocr_meta(arxiv_id, url, markdown, result)
+    meta["conversion_mode"] = "native_fallback"
+    meta["quality"] = "partial"
+    meta["ocr_error"] = str(error)
+    return meta
+
+
+def _usable(markdown: str | None) -> bool:
+    return markdown is not None and len(markdown.strip()) >= MIN_MARKDOWN_CHARS
+
+
+def _is_pdf(path: Path) -> bool:
+    try:
+        with path.open("rb") as stream:
+            return stream.read(4) == b"%PDF"
+    except OSError:
+        return False
+
+
+def _download_pdf(url: str, pdf_path: Path) -> bool:
+    raw = _fetch(url)
+    if raw is None:
+        print(f"could not fetch {url}", file=sys.stderr)
+        return False
+    if not raw.startswith(b"%PDF"):
+        print(f"{url} did not return a PDF", file=sys.stderr)
+        return False
+    descriptor, staged = tempfile.mkstemp(prefix=f".{pdf_path.name}-", suffix=".part", dir=pdf_path.parent)
+    try:
+        with os.fdopen(descriptor, "wb") as stream:
+            stream.write(raw)
+        os.replace(staged, pdf_path)
+    finally:
+        try:
+            os.unlink(staged)
+        except FileNotFoundError:
+            pass
+    return True
+
+
+def convert(arxiv_id: str, *, allow_native_fallback: bool = False) -> bool:
+    d = CACHE_DIR / arxiv_id
+    cached = _cached_conversion(d, allow_native_fallback=allow_native_fallback)
+    if cached is not None:
+        print(f"{arxiv_id}: cached at {cached}")
         return True
 
     try:
-        import pymupdf4llm
+        import pdf_inspector
     except ImportError:
         print(
             "PDF conversion needs `paperstack-cli[pdf]`; reinstall with "
@@ -56,36 +185,57 @@ def convert(arxiv_id: str) -> bool:
         return False
 
     url = f"https://arxiv.org/pdf/{arxiv_id}"
-    raw = _fetch(url)
-    if raw is None:
-        print(f"{arxiv_id}: could not fetch {url}", file=sys.stderr)
-        return False
-    if not raw.startswith(b"%PDF"):
-        print(f"{arxiv_id}: {url} did not return a PDF", file=sys.stderr)
-        return False
-
     d.mkdir(parents=True, exist_ok=True)
     pdf_path = d / "paper.pdf"
-    pdf_path.write_bytes(raw)
-    md = pymupdf4llm.to_markdown(str(pdf_path))
-    if len(md) < 500:
-        print(f"{arxiv_id}: converted to only {len(md)} chars, treat as a failure", file=sys.stderr)
+    reused_cached_pdf = _is_pdf(pdf_path)
+    if not reused_cached_pdf and not _download_pdf(url, pdf_path):
         return False
 
-    md_path.write_text(md, encoding="utf-8")
-    (d / "meta.json").write_text(
-        json.dumps(
-            {
-                "arxiv_id": arxiv_id,
-                "bytes": len(md),
-                "sha256": hashlib.sha256(md.encode("utf-8")).hexdigest(),
-                "url": url,
-                "converter": "pymupdf4llm",
-            },
-            indent=2,
-        ),
-        encoding="utf-8",
-    )
+    while True:
+        ocr_error = None
+        try:
+            result = pdf_inspector.process_pdf_with_ocr(str(pdf_path), mode="auto")
+            parsed = result
+            md = result.markdown
+            meta = _ocr_meta(arxiv_id, url, md, result)
+        except ValueError as error:
+            ocr_error = error
+            try:
+                native = pdf_inspector.process_pdf_with_ocr(str(pdf_path), mode="off")
+            except ValueError:
+                native = None
+            parsed = native
+            if native is None:
+                md = None
+            else:
+                md = native.markdown
+                if _usable(md):
+                    meta = _native_fallback_meta(arxiv_id, url, md, native, error)
+
+        if _usable(md):
+            if ocr_error is not None:
+                print(f"{arxiv_id}: OCR unavailable; cached {meta['quality']} native extraction", file=sys.stderr)
+            break
+        needs_ocr = parsed is not None and bool(parsed.pages_recommended_for_ocr)
+        if reused_cached_pdf and not needs_ocr and _download_pdf(url, pdf_path):
+            reused_cached_pdf = False
+            continue
+        if ocr_error is not None:
+            print(f"{arxiv_id}: PDF conversion failed: {ocr_error}", file=sys.stderr)
+            if needs_ocr:
+                print(
+                    "OCR needs PDFium and ONNX Runtime; set PDFIUM_LIB_PATH and ORT_DYLIB_PATH.",
+                    file=sys.stderr,
+                )
+                print(f"OCR runtime setup: {OCR_RUNTIME_GUIDE}", file=sys.stderr)
+            return False
+        chars = len(md.strip()) if md else 0
+        print(f"{arxiv_id}: converted to only {chars} chars, treat as a failure", file=sys.stderr)
+        return False
+
+    md_path = d / "paper.md"
+    md_path.write_bytes(md.encode("utf-8"))
+    (d / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
     print(f"{arxiv_id}: {len(md)} chars -> {md_path}")
     return True
 

paperstack/entrypoint.py

@@ -9,6 +9,8 @@ from dotenv import find_dotenv, load_dotenv
 
 from . import credentials
 
+_EXPORTED_ONLY = ("PDFIUM_LIB_PATH", "ORT_DYLIB_PATH")
+
 
 def load_environment() -> None:
     """Load the nearest .env without replacing exported variables."""
@@ -16,6 +18,9 @@ def load_environment() -> None:
     path = find_dotenv(usecwd=True)
     if path:
         load_dotenv(path, override=False)
+    for name in _EXPORTED_ONLY:
+        if name not in exported_keys:
+            os.environ.pop(name, None)
     credentials.set_environment_context(exported_keys, Path(path) if path else None)
 
 

{paperstack_cli-0.3.2.dist-info → paperstack_cli-0.4.0.dist-info}/METADATA

@@ -1,22 +1,29 @@
 Metadata-Version: 2.5
 Name: paperstack-cli
-Version: 0.3.2
+Version: 0.4.0
 Summary: Review, inspect, and retrieve research sources from one CLI
 Project-URL: Repository, https://github.com/MilkClouds/paperstack
 Project-URL: Issues, https://github.com/MilkClouds/paperstack/issues
 License-Expression: Apache-2.0
 License-File: LICENSE
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.11
 Requires-Dist: filelock>=3.20
 Requires-Dist: polars>=1.43
 Requires-Dist: python-dotenv>=1.2.2
 Requires-Dist: pyyaml>=6
 Provides-Extra: pdf
-Requires-Dist: pymupdf4llm>=0.0.17; extra == 'pdf'
+Requires-Dist: pdf-inspector<2,>=1.17; extra == 'pdf'
 Description-Content-Type: text/markdown
 
 # Paperstack
 
+[![CI](https://github.com/MilkClouds/paperstack/actions/workflows/ci.yml/badge.svg)](https://github.com/MilkClouds/paperstack/actions/workflows/ci.yml) [![PyPI version](https://img.shields.io/pypi/v/paperstack-cli.svg)](https://pypi.org/project/paperstack-cli/) [![Python versions](https://img.shields.io/pypi/pyversions/paperstack-cli.svg)](https://pypi.org/project/paperstack-cli/) [![License](https://img.shields.io/pypi/l/paperstack-cli.svg)](https://github.com/MilkClouds/paperstack/blob/main/LICENSE)
+
 Paperstack is an open-source CLI for maintaining, querying, and publishing an independently authored research corpus.
 The software and corpus are separate: Paperstack can be public while each corpus and its editorial judgments remain
 under its maintainer's control.
@@ -38,6 +45,18 @@ PDF conversion is optional:
 uv tool install 'paperstack-cli[pdf]'
 ```
 
+The PDF extra uses `pdf-inspector` for native extraction and selective OCR.
+Clean text PDFs need no external runtime, while routed OCR pages require the pinned PDFium and ONNX Runtime libraries described in the [OCR runtime guide](https://github.com/firecrawl/pdf-inspector/blob/main/docs/ocr-runtime.md).
+The first routed page downloads and verifies the pinned OCR model set unless an offline model directory is configured.
+When the shared libraries are not on the platform search path, point `pdf-inspector` at the extracted files:
+
+```bash
+export PDFIUM_LIB_PATH=/absolute/path/to/libpdfium.so
+export ORT_DYLIB_PATH=/absolute/path/to/libonnxruntime.so
+```
+
+The runtime guide lists the matching downloads and filenames for Linux, macOS, and Windows.
+
 ## AI agent skill
 
 Paperstack ships an [Agent Skills](https://agentskills.io)-compatible skill for Codex, Claude Code, and other

{paperstack_cli-0.3.2.dist-info → paperstack_cli-0.4.0.dist-info}/RECORD

@@ -1,19 +1,19 @@
 paperstack/__init__.py,sha256=Cnu12PizQdtp2EOJIPMV8LN-2JXdtSiPodSD3iISedg,39
 paperstack/arxiv.py,sha256=I3nNuzCTDy7QSd0nvt0J5skH6Y_qThQM5DjVh82sz3w,4514
 paperstack/citations.py,sha256=lnYmGEDiGJeAa8XUQ7P-Rzf5y3u3jmIp-R5ZoFCYrJ0,4237
-paperstack/cli.py,sha256=yrFkAdO9r7K_b-z9inNafW6-XuIGBvEu2RL2afSIe-E,47526
+paperstack/cli.py,sha256=mjQ-NfwjqT5sWt5cb_8Kkqg9fYxX2ZGiOInlb6TK0Wk,47593
 paperstack/corpora.py,sha256=14oQaQHLEFfWeX15DB-j9cnRl-1gGSYL6z-r32qGjQU,5367
 paperstack/credentials.py,sha256=-OvxwOfQwGM-SBkX8IN6J8El0vuG0_YIkTMCe5g8lYI,4848
 paperstack/dblp_build.py,sha256=VMNQv78-5jGr7-ZU2VVxKus2qtLimZHVhN-npEGe2Jw,6938
 paperstack/dblp_catalog.py,sha256=BZNV_DpHiiJKveV9m-2o5Ha6Dt9hR1knClvnn70mg0A,3311
 paperstack/dblp_index.py,sha256=FRDjMAjRxaUdsKB3FXX2mJCyW6ce-sBgUb4Pk3dSSXA,20340
 paperstack/entry_types.py,sha256=Iw9_K2_2w2Has4dOi_4fxYYFdWr-Kc_3i4Qj40-Fgjw,12540
-paperstack/entrypoint.py,sha256=tz46tgq2Ahy7FhqOKAVIZwRQdCdaIHDVM0xls69tn9c,726
+paperstack/entrypoint.py,sha256=VWHY4qb3ZwlTY6fnZvBSZ5L28IYwxNbUv5uCO0IdQRc,891
 paperstack/metadata.py,sha256=x3B0UP_yLjJBFFC0Yez-sGS5dLFCT1BS59rMzb37tnw,16345
 paperstack/semantic_scholar.py,sha256=SGJpmy4zcuwptq1Joowg_ykusyrCxA85lLFIYPvCEb8,6089
 paperstack/viewer.py,sha256=HHBi7B5fo-vTUU5ZIf1gY2qEYQ9qi76YHzl1xUUlnCs,7265
 paperstack/content/__init__.py,sha256=rPAfhJSpEpOFyKlPNbbfEZ9-oDTbfJDEqxJHnCgHVp0,54
-paperstack/content/arxiv_pdf.py,sha256=TjqpPVIVVU--T9qTJsiRSqt8xOZ_3YDcEulZEX7D6DU,3073
+paperstack/content/arxiv_pdf.py,sha256=m18-8CL0FAenJ0jo77QYbjpp5mKAbQ9hhyGOnNgPeUc,8778
 paperstack/content/arxiv_source.py,sha256=6LPMGGfi7UzKBUJglMBQWhiHVjXSrIzKRxDqjC1O_zA,15122
 paperstack/content/vendor/latexpand,sha256=Z-ylXQmz-5gt7iNE5jF_fhDp8Vg5EC5qHbOOUY-DMDw,23338
 paperstack/content/vendor/latexpand.LICENSE,sha256=hpipH1rQQwKFd1DYN_FtpxSBbYqCBBjsgCKr8YuYJhQ,1529
@@ -24,8 +24,8 @@ paperstack/viewer_assets/site/index.html,sha256=L-RUqIQODuEYxCfky-KjdYnT8VlHuo4v
 paperstack/viewer_assets/site/style.css,sha256=MdQthEKaslRDPvEVIqB6PVkS6N9030AHGMITuSaUU24,9378
 paperstack/viewer_assets/vendor/marked.LICENSE,sha256=0XnNx76Kb3YqkDWVF6gJiKaROm5OA3MhIvCeF74B9-Y,2679
 paperstack/viewer_assets/vendor/marked.min.js,sha256=k04-Nuni2gr7Gm51B1uw8JrwUpOoROhKdHfvQJEcNJo,39589
-paperstack_cli-0.3.2.dist-info/METADATA,sha256=Ajjnod2MBKxFJ0J71GipPqhK63nDoKcM1iUe4l6bVdA,7359
-paperstack_cli-0.3.2.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-paperstack_cli-0.3.2.dist-info/entry_points.txt,sha256=0716S7tM1TuT4_aj291u3335bww-f0K0wQ1u8pS7VxU,58
-paperstack_cli-0.3.2.dist-info/licenses/LICENSE,sha256=_i01vK4EN87TL8E1Fo6RsNIkMi9-_t-wzHTFiqfg9xk,11357
-paperstack_cli-0.3.2.dist-info/RECORD,,
+paperstack_cli-0.4.0.dist-info/METADATA,sha256=JvrE3kXgPf8j0_qbZmusx54-a-XdKdjVMlk7_INi0CY,8883
+paperstack_cli-0.4.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+paperstack_cli-0.4.0.dist-info/entry_points.txt,sha256=0716S7tM1TuT4_aj291u3335bww-f0K0wQ1u8pS7VxU,58
+paperstack_cli-0.4.0.dist-info/licenses/LICENSE,sha256=_i01vK4EN87TL8E1Fo6RsNIkMi9-_t-wzHTFiqfg9xk,11357
+paperstack_cli-0.4.0.dist-info/RECORD,,