fetchpdf 0.1.0__py3-none-any.whl

fetchpdf/retrieval/__init__.py

@@ -0,0 +1,36 @@
+"""Format-prioritized retrieval.
+
+The default retrieval path in ``fetch_pdf`` is PDF-shaped: it walks a
+fixed list of sources and accepts the first thing with ``%PDF`` magic bytes.
+That is the right default for a human who wants a paper to read.
+
+It is the wrong default for feeding a table-extraction pipeline, because what
+breaks there is not character accuracy but row/column/header association -- and
+that association only survives losslessly in markup. This package inverts the
+loop: format tiers outer, sources inner, so a JATS copy from a worse-ranked
+source beats a PDF from a better-ranked one.
+
+This package also holds the second, independent opt-in feature that grew out of
+the same machinery: ``supplementary``, reached by ``--pull-supplementary``, which
+fetches everything the authors deposited *alongside* the paper. It is not a tier.
+The tier walk is winner-take-all by design -- one best representation of the full
+text -- and "what else is there" is a set, not a winner, so it runs as a pass
+after retrieval rather than as a rung inside it. It shares this package's HTTP
+client, rate limiter and identifier resolver, which is why it lives here.
+
+Nothing in here runs unless one of the opt-in flags is set. With none of them,
+the existing chain executes byte-for-byte as before.
+"""
+
+from .tiers import Tier, Ladder, load_ladder
+from .identifiers import IdentifierSet, ResolutionChain
+from .artifact import Artifact
+
+__all__ = [
+    "Tier",
+    "Ladder",
+    "load_ladder",
+    "IdentifierSet",
+    "ResolutionChain",
+    "Artifact",
+]

fetchpdf/retrieval/_util.py

@@ -0,0 +1,150 @@
+"""Small helpers that more than one module in this package needs.
+
+Each of these existed in two to four copies before. That is mostly harmless
+duplication -- except for `localname`, where it was not: both copies need the
+non-string guard for lxml comment and processing-instruction nodes, whose `.tag`
+is a *callable* rather than a name. A copy written without that guard raised
+TypeError on every real publisher page, which is how T2 validation was broken
+until an integration run caught it. One definition, one guard.
+"""
+
+import html as _html
+import os
+import unicodedata
+from datetime import datetime, timezone
+from typing import Optional
+
+#: Two titles for the same work are the same title at 0.93 and are different
+#: works well below it. The margin is not delicate: a Dataverse deposit and its
+#: own article score 0.99, an unrelated one 0.12. Anything in between is not a
+#: near-miss worth arguing over. `supplement_graph` still carries its own copy
+#: of this number and its own `_squash`; they predate these and have not been
+#: migrated, so the two are free to drift. Anyone touching either should move
+#: that caller over rather than adding a third.
+TITLE_MATCH_RATIO = 0.93
+
+
+def localname(tag) -> str:
+    """An element tag without its namespace.
+
+    TEI is namespaced, JATS usually is not, and lxml hands back comments and
+    processing instructions whose `.tag` is a function -- hence the isinstance
+    check, which is the whole reason this lives in one place.
+    """
+    if not isinstance(tag, str):
+        return ""
+    return tag.rsplit("}", 1)[-1].lower() if "}" in tag else tag.lower()
+
+
+def as_int(value, default: int = 0) -> int:
+    """int(value) for values that arrive from JSON as strings, or as None."""
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def iso(timestamp: float) -> str:
+    """A UTC ISO-8601 stamp. Used in every audit record this package writes."""
+    return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def unlink(path: Optional[str]) -> None:
+    """Delete a path, tolerating its absence. Cleanup must never raise."""
+    if not path:
+        return
+    try:
+        os.unlink(path)
+    except OSError:
+        pass
+
+
+def squash(text: str) -> str:
+    """Lowercased alphanumerics only, so punctuation and casing differences
+    between a deposit title and a journal title cannot break the match.
+
+    Dropping *everything* non-alphanumeric is also what makes this usable
+    against text extracted from a PDF page: a title broken across a line, or
+    hyphenated at the break, squashes to the same string as the title it came
+    from. That is load-bearing for `pdf_identity`, not incidental.
+
+    The NFKD pass is load-bearing for the same caller and for a reason that is
+    invisible until it bites: PDF text extraction hands back the *ligature*
+    glyphs a typesetter used, so a page reading "identification" arrives as
+    "identification" -- one character, U+FB01. Without decomposition that title
+    silently fails to match itself, and under reject-always a silent non-match
+    is a deleted file. Decomposing first turns the ligature back into "fi". It
+    also strips accents to their base letters, so "café" and "cafe" agree.
+
+    Kept as `str.isalnum()` rather than `[^a-z0-9]`, which is what this was:
+    that character class erases every non-Latin script outright. A Cyrillic,
+    Greek or CJK title squashed to the empty string, and an empty title matches
+    nothing -- so under reject-always every paper with a non-Latin title and no
+    DOI printed on the page would have been deleted as somebody else's.
+    """
+    # Unescaped first because titles arrive from Crossref carrying HTML
+    # entities -- a real title reads "salience & message framing" in the
+    # metadata and "salience & message framing" on the page, and the stray
+    # "amp" splits the match straight down the middle.
+    unescaped = _html.unescape(text or "")
+    decomposed = unicodedata.normalize("NFKD", unescaped).lower()
+    return "".join(ch for ch in decomposed if ch.isalnum())
+
+
+def matched_fraction(needle: str, haystack: str, min_block: int = 4) -> float:
+    """How much of `needle` appears in `haystack`, across ALL matching runs.
+
+    The right question to ask when matching a title against a page of extracted
+    text, and `SequenceMatcher.ratio()` is the wrong one: ratio is symmetric and
+    length-sensitive, so a 90-character title compared against 4,000 characters
+    of page scores near zero however perfectly the title appears on it.
+
+    Deliberately gap-tolerant. A single substituted character in the middle of a
+    title halves the longest contiguous run while leaving the title obviously
+    intact -- measured on a real record, "V717F β-Amyloid Precursor Protein"
+    typeset as "V717F b-Amyloid Precursor Protein" scores 0.62 by longest run
+    and 0.99 by matched total. Greek letters set in a Latin face are pervasive
+    in the life sciences, so that is not an edge case, it is Tuesday.
+
+    `min_block` is what keeps this honest: without it, a long page of prose
+    supplies a matching character somewhere for nearly every letter of any
+    title, and everything scores high. Only runs of at least `min_block`
+    characters count, so the score still means "this page contains these
+    phrases", not "this page contains these letters".
+
+    Both sides are assumed already squashed.
+    """
+    import difflib
+
+    if not needle or not haystack:
+        return 0.0
+    matcher = difflib.SequenceMatcher(None, needle, haystack, autojunk=False)
+    total = sum(b.size for b in matcher.get_matching_blocks() if b.size >= min_block)
+    return min(1.0, total / float(len(needle)))
+
+
+def normalise_doi(doi: str) -> str:
+    """A DOI stripped of the several prefixes it arrives wearing."""
+    doi = (doi or "").strip().lower()
+    for prefix in ("doi:", "https://doi.org/", "http://doi.org/", "doi.org/",
+                   "https://dx.doi.org/", "http://dx.doi.org/", "dx.doi.org/"):
+        if doi.startswith(prefix):
+            doi = doi[len(prefix):]
+    return doi
+
+
+def titles_match(a: str, b: str, ratio: float = TITLE_MATCH_RATIO) -> bool:
+    """Whether two titles name the same work.
+
+    Containment first -- it is cheap and it is the common case, since one side
+    is routinely a subtitle-trimmed or prefix-wrapped form of the other. The
+    difflib ratio is only for the spelling-variant tail that containment drops.
+    """
+    import difflib
+
+    left, right = squash(a), squash(b)
+    if not left or not right:
+        return False
+    if left in right or right in left:
+        return True
+    return difflib.SequenceMatcher(None, left, right).ratio() >= ratio

fetchpdf/retrieval/artifact.py

@@ -0,0 +1,87 @@
+"""What a source returns, and what a validator says about it.
+
+Kept in its own module so sources, validators, the engine and the provenance
+writer can all share these types without importing each other.
+"""
+
+import hashlib
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from .tiers import TIER_EXTENSIONS, Tier
+
+
+@dataclass
+class ValidationResult:
+    """The outcome of a tier gate, and enough detail to say WHY.
+
+    `reason` is populated on failure and on success alike: provenance has to
+    record which checks passed, not merely that some unnamed check did.
+    """
+
+    ok: bool
+    reason: str = ""
+    checks_passed: List[str] = field(default_factory=list)
+    n_chars: int = 0
+    n_tables: int = 0
+    n_footnotes: int = 0
+    n_populated_cells: int = 0
+
+    def to_dict(self) -> dict:
+        return {
+            "ok": self.ok,
+            "reason": self.reason,
+            "checks_passed": list(self.checks_passed),
+            "n_chars": self.n_chars,
+            "n_tables": self.n_tables,
+            "n_footnotes": self.n_footnotes,
+            "n_populated_cells": self.n_populated_cells,
+        }
+
+    @classmethod
+    def failure(cls, reason: str, **kwargs) -> "ValidationResult":
+        return cls(ok=False, reason=reason, **kwargs)
+
+
+@dataclass
+class Artifact:
+    """One retrieved candidate, before or after validation.
+
+    `tier` is what the source expected to produce. The engine overwrites it with
+    the tier the classifier assigns from the bytes themselves -- a source that
+    advertises XML and returns an HTML error page is reclassified, not trusted.
+    """
+
+    content: bytes
+    tier: Tier
+    source: str
+    url: str
+    http_status: int
+    served_content_type: str = ""
+    identifier_used: str = ""
+    license: Optional[str] = None
+    extension: Optional[str] = None
+    validation: Optional[ValidationResult] = None
+    tables_unavailable: bool = False
+    declared_tier: Optional[Tier] = None      # what the source claimed, if reclassified
+    normalization_failures: List[str] = field(default_factory=list)
+    extra: Dict[str, Any] = field(default_factory=dict)
+
+
+
+    @property
+    def suffix(self) -> str:
+        return self.extension or TIER_EXTENSIONS[self.tier]
+
+    @property
+    def content_hash(self) -> str:
+        return "sha256:" + hashlib.sha256(self.content).hexdigest()
+
+    def __len__(self):
+        return len(self.content)
+
+    def __repr__(self):
+        return (
+            f"<Artifact {self.source} {self.tier.name} "
+            f"{len(self.content)}B status={self.http_status}>"
+        )

fetchpdf/retrieval/backends/__init__.py

@@ -0,0 +1,109 @@
+"""Backends for the retrieval agent: one contract, two very different sandboxes.
+
+The agent's job is the same either way -- read the paper, work out which
+supplementary files and datasets we do not already have, and get them -- but
+what it is *allowed to do* differs so much between the two that the difference
+is the interesting part.
+
+MEASURED 2026-08-22, against Claude Code 2.1.240, because the containment story
+in the design turned out to be wrong and only testing found it:
+
+  * `--allowed-tools` GRANTS permission, it does not restrict the tool set. A
+    subprocess run with `--allowed-tools "Bash(echo:*)"` happily ran
+    `cat ../outside_canary.txt` and returned the contents. So did
+    `--allowed-tools WebFetch`. Both with and without
+    `--permission-mode bypassPermissions`.
+  * `--disallowed-tools` DOES remove tools, and is the only lever that worked.
+  * A default subprocess inherits the operator's own MCP servers. On the
+    machine this was written on that meant Gmail, Google Drive, Google Calendar
+    and a brokerage account were in scope for a subprocess whose job is
+    downloading spreadsheets. `--strict-mcp-config` with an empty config
+    removes them.
+
+The consequence: the CLI backend gets WebFetch and nothing else -- no shell, no
+file tools, no MCP -- and therefore *cannot write a file*. So it does not
+download. It navigates, which is the part that actually needs a model (open the
+landing page, find the real file link, decide whether this deposit belongs to
+this paper), and reports URLs. fetchpdf does the transfer, through the same
+capped, hashed, deduplicated path every other provider uses.
+
+The OpenRouter backend is the opposite: there is no general-purpose agent
+harness to lock down, because the two tools it can call are ours. `download`
+writes inside the staging directory by construction and enforces the byte cap
+itself. That backend really does download.
+
+Both return the same receipt, and the caller cannot tell which one ran by
+looking at the output directory. That is the point.
+"""
+
+from typing import Optional
+
+#: The receipt one entry at a time. Keys a backend may set:
+#:
+#:   url        the direct file URL, or the deposit landing page
+#:   kind       "supplement" (the paper's own SI) or "dataset" (a deposit)
+#:   deposit    the repository landing page this file came from, if any
+#:   name       the filename to record it under, when the URL does not say
+#:   why        one short line: why this belongs to THIS paper
+#:   saved_as   staging-relative path, set only by a backend that downloads
+RECEIPT_KEYS = ("url", "kind", "deposit", "name", "why", "saved_as")
+
+KIND_SUPPLEMENT = "supplement"
+KIND_DATASET = "dataset"
+
+#: Registered backends, resolved by `--llm-backend`.
+BACKENDS = ("claude-cli", "openrouter")
+DEFAULT_BACKEND = "claude-cli"
+
+
+def get_backend(name: Optional[str] = None):
+    """A backend instance by name, or None when the name is unknown.
+
+    Imported lazily so a run that never asks for an agent never imports one.
+    """
+    name = (name or DEFAULT_BACKEND).strip().lower()
+    if name == "claude-cli":
+        from .claude_cli import ClaudeCliBackend
+        return ClaudeCliBackend()
+    if name == "openrouter":
+        from .openrouter import OpenRouterBackend
+        return OpenRouterBackend()
+    return None
+
+
+def normalize_receipt(raw, staging: str = "") -> list:
+    """Whatever the model returned, reduced to entries we can act on.
+
+    Defensive by policy rather than by taste: this is the one place a model's
+    free-form output crosses into code that downloads things. An entry without
+    a usable URL or staged file is dropped rather than guessed at, and a
+    `saved_as` that escapes the staging directory is dropped too -- a path is
+    the one field a model could use to reach outside its sandbox.
+    """
+    import os
+
+    if not isinstance(raw, list):
+        return []
+    staging_real = os.path.realpath(staging) if staging else ""
+    entries = []
+    for item in raw:
+        if not isinstance(item, dict):
+            continue
+        entry = {k: item.get(k) for k in RECEIPT_KEYS if item.get(k)}
+        saved = entry.get("saved_as")
+        if saved:
+            candidate = os.path.realpath(os.path.join(staging, str(saved)))
+            inside = (staging_real and
+                      (candidate == staging_real or
+                       candidate.startswith(staging_real + os.sep)))
+            if not inside or not os.path.isfile(candidate):
+                entry.pop("saved_as", None)
+            else:
+                entry["saved_as"] = candidate
+        url = str(entry.get("url") or "")
+        if not entry.get("saved_as") and not url.lower().startswith(("http://", "https://")):
+            continue
+        kind = str(entry.get("kind") or "").lower()
+        entry["kind"] = KIND_DATASET if kind.startswith("data") else KIND_SUPPLEMENT
+        entries.append(entry)
+    return entries

fetchpdf/retrieval/backends/claude_cli.py

@@ -0,0 +1,111 @@
+"""The retrieval agent on the local Claude Code CLI, locked down to WebFetch.
+
+Reuses `llm_adjudicate`'s subprocess plumbing -- `find_cli`, `run_cli`,
+`unwrap_result`, `_warn_once` -- so there is one place that knows about the
+Windows .cmd shim, the stdin feed and the JSON envelope.
+
+WHY IT DOES NOT DOWNLOAD. See `backends/__init__.py` for the measurements: in
+this CLI, `--allowed-tools` grants and does not restrict, and a default
+subprocess inherits the operator's MCP servers. The only configuration that
+verifiably contained the agent removes every file and shell tool -- which also
+removes its ability to save anything. That is a fair trade. Navigating is the
+part that needs a model; transferring bytes under a cap with a hash is the part
+fetchpdf already does better than a shell command would.
+
+The lockdown is a DENYLIST, and denylists rot: a tool added to Claude Code in a
+later release is allowed here until someone adds it below. The two structural
+guards are what this actually rests on -- no MCP config, no settings sources --
+plus the fact that a WebFetch-only agent has nothing to write to.
+"""
+
+import json
+import os
+import tempfile
+from typing import List, Optional
+
+from ..llm_adjudicate import (
+    DEFAULT_MODEL,
+    _warn_once,
+    find_cli,
+    run_cli,
+    unwrap_result,
+)
+from . import normalize_receipt
+
+#: The one tool it needs, and every built-in that could read the operator's
+#: disk, spawn another agent, or reach a connected service. Verified against
+#: Claude Code 2.1.240 by asking a locked-down subprocess to list its own
+#: tools: it answered "WebFetch" and nothing else.
+ALLOWED_TOOLS = "WebFetch"
+DISALLOWED_TOOLS = ",".join((
+    "Bash", "Read", "Write", "Edit", "NotebookEdit", "Glob", "Grep",
+    "Agent", "Skill", "Workflow", "SendMessage", "ListAgents",
+    "TaskCreate", "TaskGet", "TaskList", "TaskOutput", "TaskStop", "TaskUpdate",
+    "WebSearch", "Monitor", "ScheduleWakeup", "ReportFindings", "ToolSearch",
+    "CronCreate", "CronDelete", "CronList", "DesignSync",
+    "EnterWorktree", "ExitWorktree", "PushNotification", "RemoteTrigger",
+    "ListMcpResourcesTool", "ReadMcpResourceTool", "ReadMcpResourceDirTool",
+))
+
+#: Enough turns to open a landing page, follow it to a file listing, and
+#: answer. Measured: a single OSF node lookup takes 2. The cap IS enforced --
+#: a run that exceeds it comes back is_error with no result at all, which is
+#: why this is generous rather than tight.
+MAX_TURNS = 12
+
+#: Wall clock for the whole conversation. The adjudicator's 60s is sized for
+#: one classification; this one makes several network round trips.
+TIMEOUT_SECONDS = 300
+
+
+class ClaudeCliBackend:
+    """Navigates with WebFetch and reports URLs. Never writes a file."""
+
+    name = "claude-cli"
+    downloads = False
+
+    def available(self):
+        """`(ok, detail)`. Absence is a reason, never an exception."""
+        if find_cli() is None:
+            return False, ("Claude Code CLI not found on PATH; install it or "
+                           "use --llm-backend openrouter")
+        return True, "claude"
+
+    def run(self, brief: str, staging: str, model: str = DEFAULT_MODEL,
+            log=None) -> Optional[List[dict]]:
+        cli_path = find_cli()
+        if cli_path is None:
+            ok, detail = self.available()
+            _warn_once(detail, log)
+            return None
+
+        with tempfile.TemporaryDirectory() as empty_config_dir:
+            config = os.path.join(empty_config_dir, "no_mcp.json")
+            with open(config, "w", encoding="utf-8") as handle:
+                json.dump({"mcpServers": {}}, handle)
+
+            stdout = run_cli(
+                brief,
+                cli_path,
+                model=model,
+                timeout=TIMEOUT_SECONDS,
+                allowed_tools=ALLOWED_TOOLS,
+                max_turns=MAX_TURNS,
+                extra_argv=[
+                    # Structural, not a denylist: without these the subprocess
+                    # inherits every MCP server the operator has connected.
+                    "--strict-mcp-config", "--mcp-config", config,
+                    "--setting-sources", "",
+                    "--disallowed-tools", DISALLOWED_TOOLS,
+                    # Nothing can prompt in print mode, so an un-preapproved
+                    # WebFetch would simply be denied and the agent would
+                    # report nothing. The tool set above is the boundary.
+                    "--permission-mode", "bypassPermissions",
+                ],
+                cwd=staging,
+                log=log,
+            )
+
+        if stdout is None:
+            return None
+        return normalize_receipt(unwrap_result(stdout, expect=list), staging)