paperstack-cli 0.4.1__tar.gz → 0.5.0__tar.gz

{paperstack_cli-0.4.1 → paperstack_cli-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: paperstack-cli
-Version: 0.4.1
+Version: 0.5.0
 Summary: Review, inspect, and retrieve research sources from one CLI
 Project-URL: Repository, https://github.com/MilkClouds/paperstack
 Project-URL: Issues, https://github.com/MilkClouds/paperstack/issues
@@ -146,12 +146,25 @@ paperstack paper read arxiv:2604.23073 --section 6
 paperstack paper pdf arxiv:2602.09017
 ```
 
-`metadata` accepts `arxiv:`, `doi:`, `dblp:`, and `openreview:` references. `read` and `pdf` require an `arxiv:`
-reference. `authors`, `citations`, and `references` use Semantic Scholar and also accept its `s2:`, `corpus:`, `acl:`,
+`metadata` accepts `arxiv:`, `doi:`, `dblp:`, and `openreview:` references. `read` and `pdf`
+accept arXiv references directly; other identifiers require an explicit primary-source HTTPS `--pdf-url`. `authors`, `citations`, and `references` use Semantic Scholar and also accept its `s2:`, `corpus:`, `acl:`,
 `pmid:`, and `mag:` identifiers. Structured commands expose scoped `--json` flags; networked commands expose scoped
 `--offline` flags. Paperstack reports source records but does not synthesize a citation entry or choose which version
 of a work should be cited.
 
+### Source reading
+
+- Pin `arxiv:IDvN` for version-specific downloads and caches. Metadata lookup remains work-level.
+  Unversioned reads may use stale cached content; `--refresh` refetches, `--offline` never does.
+- `read ... --documents` lists source files. Multiple TeX roots require `--document FILE`;
+  inspect relevant supplements separately. `--outline`/`--section` apply to that document.
+- DOI/OpenReview/DBLP reads require `--pdf-url https://...` pointing to the primary PDF.
+  `read` prints Markdown; `pdf` prints its cache path. New conversions need the PDF extra.
+- Partial PDF extraction returns failure unless `--allow-partial` is explicit, even offline.
+  Check `meta.json` and the PDF before using numbers. Converter quality does not certify accuracy.
+- PDF caches verify original and Markdown hashes. Old caches without PDF hashes need rebuilding.
+
+
 OpenReview exact-title search uses its title-only exact mode and verifies a normalized title match locally. Status
 filtering is conservative because venues encode decisions and withdrawals differently; Paperstack infers it from
 public invitation, venue, decision, and status fields rather than treating it as a universal field.

{paperstack_cli-0.4.1 → paperstack_cli-0.5.0}/README.md

@@ -124,12 +124,25 @@ paperstack paper read arxiv:2604.23073 --section 6
 paperstack paper pdf arxiv:2602.09017
 ```
 
-`metadata` accepts `arxiv:`, `doi:`, `dblp:`, and `openreview:` references. `read` and `pdf` require an `arxiv:`
-reference. `authors`, `citations`, and `references` use Semantic Scholar and also accept its `s2:`, `corpus:`, `acl:`,
+`metadata` accepts `arxiv:`, `doi:`, `dblp:`, and `openreview:` references. `read` and `pdf`
+accept arXiv references directly; other identifiers require an explicit primary-source HTTPS `--pdf-url`. `authors`, `citations`, and `references` use Semantic Scholar and also accept its `s2:`, `corpus:`, `acl:`,
 `pmid:`, and `mag:` identifiers. Structured commands expose scoped `--json` flags; networked commands expose scoped
 `--offline` flags. Paperstack reports source records but does not synthesize a citation entry or choose which version
 of a work should be cited.
 
+### Source reading
+
+- Pin `arxiv:IDvN` for version-specific downloads and caches. Metadata lookup remains work-level.
+  Unversioned reads may use stale cached content; `--refresh` refetches, `--offline` never does.
+- `read ... --documents` lists source files. Multiple TeX roots require `--document FILE`;
+  inspect relevant supplements separately. `--outline`/`--section` apply to that document.
+- DOI/OpenReview/DBLP reads require `--pdf-url https://...` pointing to the primary PDF.
+  `read` prints Markdown; `pdf` prints its cache path. New conversions need the PDF extra.
+- Partial PDF extraction returns failure unless `--allow-partial` is explicit, even offline.
+  Check `meta.json` and the PDF before using numbers. Converter quality does not certify accuracy.
+- PDF caches verify original and Markdown hashes. Old caches without PDF hashes need rebuilding.
+
+
 OpenReview exact-title search uses its title-only exact mode and verifies a normalized title match locally. Status
 filtering is conservative because venues encode decisions and withdrawals differently; Paperstack infers it from
 public invitation, venue, decision, and status fields rather than treating it as a universal field.

{paperstack_cli-0.4.1 → paperstack_cli-0.5.0}/src/paperstack/cli.py

@@ -872,6 +872,43 @@ def _paper_cache() -> Path:
     return Path(os.environ.get("PAPERSTACK_PAPERS_DIR", base / "paperstack" / "papers"))
 
 
+def _read_pdf(a, ref, offline):
+    import hashlib
+    from contextlib import redirect_stdout
+    from urllib.parse import urlparse
+
+    from .content import arxiv_pdf
+    from .content.arxiv_source import _print_chunk
+
+    url = a.pdf_url
+    if ref.kind != "arxiv" and not url:
+        die("non-arXiv content requires --pdf-url with the primary-source PDF URL")
+    if url and (urlparse(url).scheme != "https" or not urlparse(url).hostname or urlparse(url).username):
+        die("--pdf-url must be an HTTPS URL without credentials")
+    if offline and a.refresh:
+        die("--offline and --refresh cannot be used together")
+    if a.paper_cmd == "read" and (a.outline or a.section_id or a.documents or a.document):
+        die("PDF reads support --start/--max-chars, not LaTeX document or section selection")
+    key = ref.value if not url else "external-" + hashlib.sha256(f"{ref.kind}:{ref.value}\n{url}".encode()).hexdigest()
+    arxiv_pdf.CACHE_DIR = _paper_cache()
+    directory = arxiv_pdf.CACHE_DIR / key
+    if offline:
+        if arxiv_pdf._cached_conversion(directory, allow_partial=a.allow_partial) is None:
+            die("no usable cached PDF conversion; partial content requires --allow-partial")
+    else:
+        with redirect_stdout(sys.stderr):
+            converted = arxiv_pdf.convert(
+                key, url=url, refresh=a.refresh, allow_partial=a.allow_partial, paper_ref=f"{ref.kind}:{ref.value}"
+            )
+        if not converted:
+            return 1
+    if a.paper_cmd == "read":
+        _print_chunk((directory / "paper.md").read_text(), a.start, a.max_chars)
+    else:
+        print(directory / "paper.md")
+    return 0
+
+
 def _run_paper(a: argparse.Namespace) -> int:
     from . import credentials, metadata
 
@@ -1008,44 +1045,35 @@ def _run_paper(a: argparse.Namespace) -> int:
         return 0 if result["status"] == "ok" else 1
 
     try:
-        ref = metadata.PaperRef.parse(a.paper_ref)
+        ref = metadata.PaperRef.parse(a.paper_ref, preserve_version=True)
     except ValueError as exc:
         die(str(exc))
-    if ref.kind != "arxiv":
-        die(f"paper {a.paper_cmd} currently requires an arxiv: reference")
-    if a.paper_cmd == "read":
-        from .content import arxiv_source
-
-        arxiv_source.CACHE_DIR = _paper_cache()
-        cached_source = arxiv_source.CACHE_DIR / ref.value / "src"
-        if offline and a.refresh:
-            die("--offline and --refresh cannot be used together")
-        if offline and (not cached_source.is_dir() or not arxiv_source._tex_candidates(cached_source)):
-            die(f"no complete cached source for arxiv:{ref.value}")
-        argv = ["read", ref.value]
-        if a.outline:
-            argv.append("--outline")
-        elif a.section_id:
-            argv.extend(["--section", a.section_id])
-        if a.refresh:
-            argv.append("--refresh")
-        argv.extend(["--start", str(a.start), "--max-chars", str(a.max_chars)])
-        arxiv_source.main(argv)
-        return 0
-    if a.paper_cmd == "pdf":
-        from .content import arxiv_pdf
-
-        arxiv_pdf.CACHE_DIR = _paper_cache()
-        cached_pdf = arxiv_pdf._cached_conversion(
-            arxiv_pdf.CACHE_DIR / ref.value,
-            allow_native_fallback=offline,
-        )
-        if offline and cached_pdf is None:
-            die(f"no usable cached PDF conversion for arxiv:{ref.value}")
-        if not arxiv_pdf.convert(ref.value, allow_native_fallback=offline):
-            return 1
-        return 0
-    raise AssertionError(a.paper_cmd)
+    if ref.kind == "arxiv" and not re.search(r"v[1-9]\d*$", ref.value):
+        warn("Unversioned arXiv reference: cached content may not be current; pin vN for reproducible reads.")
+    if a.pdf_url or a.paper_cmd == "pdf" or ref.kind != "arxiv":
+        return _read_pdf(a, ref, offline)
+    from .content import arxiv_source
+
+    arxiv_source.CACHE_DIR = _paper_cache()
+    cached_source = arxiv_source.CACHE_DIR / ref.value / "src"
+    if offline and a.refresh:
+        die("--offline and --refresh cannot be used together")
+    if offline and (not cached_source.is_dir() or not arxiv_source._tex_candidates(cached_source)):
+        die(f"no complete cached source for arxiv:{ref.value}")
+    argv = ["read", ref.value]
+    if a.outline:
+        argv.append("--outline")
+    elif a.section_id:
+        argv.extend(["--section", a.section_id])
+    if a.documents:
+        argv.append("--documents")
+    if a.document:
+        argv.extend(["--document", a.document])
+    if a.refresh:
+        argv.append("--refresh")
+    argv.extend(["--start", str(a.start), "--max-chars", str(a.max_chars)])
+    arxiv_source.main(argv)
+    return 0
 
 
 def _run_index(a: argparse.Namespace) -> int:
@@ -1226,16 +1254,23 @@ Use `paperstack review ...` to find or read an authored critical judgment.""",
         _output(s)
         _offline(s)
     s = paper_sub.add_parser("read", help="read the LaTeX body, outline, or one section")
-    s.add_argument("paper_ref", help="arxiv: reference")
+    s.add_argument("paper_ref", help="arxiv: (version preserved), doi:, dblp:, or openreview: reference")
     mode = s.add_mutually_exclusive_group()
     mode.add_argument("--outline", action="store_true", help="print numbered section headings only")
     mode.add_argument("--section", dest="section_id", help="print one section by outline number")
     s.add_argument("--refresh", action="store_true", help="replace the cached arXiv source")
     s.add_argument("--start", type=int, default=0, help="start at this character offset")
     s.add_argument("--max-chars", type=int, default=0, help="truncate output after this many characters")
+    s.add_argument("--documents", action="store_true", help="list source files, including supplements")
+    s.add_argument("--document", help="source-relative TeX root to read")
+    s.add_argument("--pdf-url", help="explicit primary-source HTTPS PDF URL")
+    s.add_argument("--allow-partial", action="store_true", help="explicitly accept incomplete PDF extraction")
     _offline(s)
     s = paper_sub.add_parser("pdf", help="download and convert a native PDF submission")
-    s.add_argument("paper_ref", help="arxiv: reference")
+    s.add_argument("paper_ref", help="arxiv: (version preserved), doi:, dblp:, or openreview: reference")
+    s.add_argument("--pdf-url", help="explicit primary-source HTTPS PDF URL")
+    s.add_argument("--allow-partial", action="store_true", help="explicitly accept incomplete PDF extraction")
+    s.add_argument("--refresh", action="store_true", help="download and convert the PDF again")
     _offline(s)
 
     index = sub.add_parser("index", help="optional local lookup indexes")

{paperstack_cli-0.4.1 → paperstack_cli-0.5.0}/src/paperstack/content/arxiv_pdf.py

@@ -16,6 +16,8 @@ import urllib.error
 import urllib.request
 from pathlib import Path
 
+from ..tls import arxiv_context
+
 _CACHE_ROOT = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
 CACHE_DIR = Path(os.environ.get("PAPERSTACK_PAPERS_DIR", _CACHE_ROOT / "paperstack" / "papers"))
 CONVERTER = "pdf-inspector"
@@ -27,7 +29,7 @@ def _fetch(url: str, timeout: int = 60) -> bytes | None:
     req = urllib.request.Request(url, headers={"User-Agent": "paperstack/1.0 (+arxiv pdf fetch)"})
     for attempt in range(3):
         try:
-            with urllib.request.urlopen(req, timeout=timeout) as resp:
+            with urllib.request.urlopen(req, timeout=timeout, context=arxiv_context()) as resp:
                 return resp.read()
         except urllib.error.HTTPError as e:
             if e.code == 429:
@@ -43,7 +45,7 @@ def _fetch(url: str, timeout: int = 60) -> bytes | None:
     return None
 
 
-def _cached_conversion(d: Path, *, allow_native_fallback: bool = False) -> Path | None:
+def _cached_conversion(d: Path, *, allow_partial: bool = False) -> Path | None:
     md_path = d / "paper.md"
     meta_path = d / "meta.json"
     if not md_path.is_file() or md_path.stat().st_size < MIN_MARKDOWN_CHARS or not meta_path.is_file():
@@ -59,7 +61,9 @@ def _cached_conversion(d: Path, *, allow_native_fallback: bool = False) -> Path
         meta.get("converter") == CONVERTER
         and meta.get("bytes") == len(markdown)
         and meta.get("sha256") == hashlib.sha256(markdown).hexdigest()
-        and (allow_native_fallback or meta.get("conversion_mode") != "native_fallback")
+        and (allow_partial or (meta.get("quality") == "complete" and meta.get("conversion_mode") != "native_fallback"))
+        and (d / "paper.pdf").is_file()
+        and meta.get("pdf_sha256") == hashlib.sha256((d / "paper.pdf").read_bytes()).hexdigest()
     )
     return md_path if valid else None
 
@@ -167,10 +171,17 @@ def _download_pdf(url: str, pdf_path: Path) -> bool:
     return True
 
 
-def convert(arxiv_id: str, *, allow_native_fallback: bool = False) -> bool:
+def convert(
+    arxiv_id: str,
+    *,
+    allow_partial: bool = False,
+    url: str | None = None,
+    refresh: bool = False,
+    paper_ref: str | None = None,
+) -> bool:
     d = CACHE_DIR / arxiv_id
-    cached = _cached_conversion(d, allow_native_fallback=allow_native_fallback)
-    if cached is not None:
+    cached = _cached_conversion(d, allow_partial=allow_partial)
+    if cached is not None and not refresh:
         print(f"{arxiv_id}: cached at {cached}")
         return True
 
@@ -184,10 +195,10 @@ def convert(arxiv_id: str, *, allow_native_fallback: bool = False) -> bool:
         )
         return False
 
-    url = f"https://arxiv.org/pdf/{arxiv_id}"
+    url = url or f"https://arxiv.org/pdf/{arxiv_id}"
     d.mkdir(parents=True, exist_ok=True)
     pdf_path = d / "paper.pdf"
-    reused_cached_pdf = _is_pdf(pdf_path)
+    reused_cached_pdf = not refresh and _is_pdf(pdf_path)
     if not reused_cached_pdf and not _download_pdf(url, pdf_path):
         return False
 
@@ -233,10 +244,21 @@ def convert(arxiv_id: str, *, allow_native_fallback: bool = False) -> bool:
         print(f"{arxiv_id}: converted to only {chars} chars, treat as a failure", file=sys.stderr)
         return False
 
+    if paper_ref:
+        meta["paper_ref"] = paper_ref
+        if not paper_ref.startswith("arxiv:"):
+            meta.pop("arxiv_id", None)
+    meta["pdf_sha256"] = hashlib.sha256(pdf_path.read_bytes()).hexdigest()
     md_path = d / "paper.md"
     md_path.write_bytes(md.encode("utf-8"))
     (d / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
     print(f"{arxiv_id}: {len(md)} chars -> {md_path}")
+    if meta["quality"] != "complete" and not allow_partial:
+        print(
+            "Partial extraction retained; inspect meta.json and the PDF, or explicitly use --allow-partial.",
+            file=sys.stderr,
+        )
+        return False
     return True
 
 

{paperstack_cli-0.4.1 → paperstack_cli-0.5.0}/src/paperstack/content/arxiv_source.py

@@ -22,6 +22,8 @@ import urllib.request
 import zlib
 from pathlib import Path
 
+from ..tls import arxiv_context
+
 _CACHE_ROOT = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
 CACHE_DIR = Path(os.environ.get("PAPERSTACK_PAPERS_DIR", _CACHE_ROOT / "paperstack" / "papers"))
 
@@ -43,7 +45,7 @@ def _fetch_bytes(url: str, timeout: int = 60) -> bytes | None:
     )
     for attempt in range(3):
         try:
-            with urllib.request.urlopen(req, timeout=timeout) as resp:
+            with urllib.request.urlopen(req, timeout=timeout, context=arxiv_context()) as resp:
                 return resp.read()
         except urllib.error.HTTPError as e:
             if e.code == 429:
@@ -323,34 +325,24 @@ def _clean_title(s: str, macros: dict[str, str] | None = None) -> str:
     return re.sub(r"\s+", " ", s).strip()
 
 
-def _load_document(arxiv_id: str, refresh: bool) -> tuple[str, list[dict]]:
-    src = _ensure_source(arxiv_id, refresh)
+def _load_document(arxiv_id: str, refresh: bool, document: str | None = None) -> tuple[str, list[dict]]:
+    src = _ensure_source(arxiv_id, refresh).resolve()
     cands = _tex_candidates(src)
-    scored = []
-    for p in cands:
-        head = _mask_comments(_read(p)[:200_000])
-        if r"\documentclass" not in head:
-            continue
-        flat = _flatten(p, src)
-        if flat is None:
-            continue
-        sections = _find_sections(flat)
-        scored.append((r"\begin{document}" in head, len(sections), -len(p.parts), flat, sections))
-    if not scored:  # Fall back when no file declares a document class.
-        for p in cands:
-            flat = _flatten(p, src)
-            if flat is None:
-                continue
-            sections = _find_sections(flat)
-            scored.append((False, len(sections), -len(p.parts), flat, sections))
-    if not scored:
+    roots = [p for p in cands if r"\documentclass" in _mask_comments(_read(p)[:200_000])]
+    if document:
+        selected = (src / document).resolve()
+        if not selected.is_relative_to(src.resolve()) or selected not in [p.resolve() for p in cands]:
+            sys.exit(f"unknown source document: {document}")
+        cands = [selected]
+    elif len(roots or cands) > 1:
+        names = ", ".join(str(p.relative_to(src)) for p in (roots or cands))
+        sys.exit(f"multiple source documents: {names}; select --document (inspect each supplement separately)")
+    selected = cands[0] if document else (roots or cands)[0] if cands else None
+    flat = _flatten(selected, src) if selected else None
+    if flat is None:
         sys.exit(f"{arxiv_id}: no usable .tex file in {src}")
-    best = max(scored, key=lambda t: (t[0], t[1], t[2]))
-    if not best[4]:
-        sys.exit(f"{arxiv_id}: LaTeX source found but no \\section commands in it")
-    masked = _mask_comments(best[3])
-    lo, hi = _body_span(masked)
-    return best[3][lo:hi], best[4]
+    lo, hi = _body_span(_mask_comments(flat))
+    return flat[lo:hi], _find_sections(flat)
 
 
 def _load(arxiv_id: str, refresh: bool) -> list[dict]:
@@ -387,7 +379,12 @@ def cmd_section(args: argparse.Namespace) -> None:
 
 
 def cmd_read(args: argparse.Namespace) -> None:
-    body, sections = _load_document(args.arxiv_id, args.refresh)
+    if getattr(args, "documents", False):
+        src = _ensure_source(args.arxiv_id, args.refresh)
+        for path in _tex_candidates(src):
+            print(path.relative_to(src))
+        return
+    body, sections = _load_document(args.arxiv_id, args.refresh, getattr(args, "document", None))
     if args.outline:
         for section in sections:
             print(f"{section['id']}\t{section['level']}\t{section['title']}")
@@ -426,6 +423,8 @@ def main(argv: list[str]) -> None:
     mode.add_argument("--section", dest="section_id", help="dotted id (3.2) or exact title")
     p_read.add_argument("--max-chars", type=int, default=0, help="0 = all remaining text")
     p_read.add_argument("--start", type=int, default=0)
+    p_read.add_argument("--documents", action="store_true")
+    p_read.add_argument("--document")
     p_read.add_argument("--refresh", action="store_true", help="refetch even if cached")
     p_read.set_defaults(func=cmd_read)
 

{paperstack_cli-0.4.1 → paperstack_cli-0.5.0}/src/paperstack/metadata.py

@@ -20,6 +20,7 @@ from pathlib import Path
 from filelock import FileLock
 
 from . import credentials
+from .tls import arxiv_context
 
 ARXIV_NS = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"}
 SOURCES = ("semantic_scholar", "dblp", "crossref", "openreview", "acl_anthology", "arxiv")
@@ -36,18 +37,21 @@ class PaperRef:
     value: str
 
     @classmethod
-    def parse(cls, raw: str) -> PaperRef:
+    def parse(cls, raw: str, *, preserve_version: bool = False) -> PaperRef:
         if ":" not in raw:
             raise ValueError("paper reference needs a prefix: arxiv:, doi:, dblp:, or openreview:")
         kind, value = raw.strip().split(":", 1)
         if kind not in ("arxiv", "doi", "dblp", "openreview") or not value:
             raise ValueError("paper reference needs a prefix: arxiv:, doi:, dblp:, or openreview:")
         if kind == "arxiv":
-            value = re.sub(r"v\d+$", "", value)
+            versioned = value
+            value = re.sub(r"v[1-9]\d*$", "", value)
             modern = re.fullmatch(r"\d{2}(?:0[1-9]|1[0-2])\.\d{4,5}", value)
             legacy = re.fullmatch(r"[A-Za-z][A-Za-z.-]*/\d{2}(?:0[1-9]|1[0-2])\d{3}", value)
             if not (modern or legacy):
                 raise ValueError("invalid arXiv reference")
+            if preserve_version:
+                value = versioned
         return cls(kind, value)
 
 
@@ -123,6 +127,7 @@ def request(
         **(headers or {}),
     }
     req = urllib.request.Request(url, headers=request_headers, data=data)
+    open_options = {"context": arxiv_context()} if "arxiv.org" in host else {}
     with _request_slot(request_key) as shared_timestamp:
         for attempt in range(3):
             elapsed = min(
@@ -132,7 +137,7 @@ def request(
             if elapsed < interval:
                 time.sleep(interval - elapsed)
             try:
-                with urllib.request.urlopen(req, timeout=30) as response:
+                with urllib.request.urlopen(req, timeout=30, **open_options) as response:
                     _last_request[request_key] = time.monotonic()
                     _mark_request(shared_timestamp)
                     return response.read()

paperstack_cli-0.5.0/src/paperstack/tls.py

@@ -0,0 +1,15 @@
+"""TLS settings for hosts that reject some client handshakes."""
+
+from __future__ import annotations
+
+import functools
+import ssl
+
+
+@functools.cache
+def arxiv_context() -> ssl.SSLContext:
+    # arXiv's Fastly edge answers uncached requests with 406 when the TLS 1.3
+    # ClientHello offers the X25519MLKEM768 hybrid key share (OpenSSL >= 3.5 default).
+    context = ssl.create_default_context()
+    context.set_ecdh_curve("X25519")
+    return context