fetchpdf 0.1.0__py3-none-any.whl

fetchpdf/retrieval/identifiers.py

@@ -0,0 +1,149 @@
+"""The identifier set built by phase 1, and the audit trail of how it was built.
+
+Resolution is separated from retrieval because a PMCID discovered via the ID
+Converter is what unlocks the entire T1 tier -- and on a real clinical corpus it
+does so for roughly 40% of records without touching a single quota-limited API.
+Interleaving resolution with retrieval per source throws that away: you cannot
+short-circuit on an identifier you have not looked up yet.
+
+IdentifierSet is mutable rather than frozen. Resolution is incremental by nature
+(stage 1 yields a PMCID, stage 3 may yield an arXiv id that feeds a second
+lookup), and threading dataclasses.replace() through five resolvers obscures
+what is actually happening. Mutation goes through learn(), which records it.
+"""
+
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class ResolutionStep:
+    """One resolution attempt, recorded whether or not it yielded anything."""
+
+    step: str                      # "crossref", "idconv", "openalex", ...
+    url: Optional[str]             # redacted before it is ever persisted
+    http_status: Optional[int]
+    yielded: Dict[str, str]        # {"pmcid": "PMC123"} -- empty dict on a miss
+    note: str = ""
+    timestamp: float = field(default_factory=time.time)
+
+    def to_dict(self) -> dict:
+        return {
+            "step": self.step,
+            "url": self.url,
+            "http_status": self.http_status,
+            "yielded": dict(self.yielded),
+            "note": self.note,
+            "timestamp": self.timestamp,
+        }
+
+
+class ResolutionChain:
+    """Ordered record of every resolution attempt for one record.
+
+    Built as resolution happens rather than reconstructed afterwards: the
+    provenance requirement is to explain why a given number came from a given
+    file, and "which lookup produced the identifier we retrieved by" is half of
+    that answer.
+    """
+
+    def __init__(self):
+        self.steps: List[ResolutionStep] = []
+
+    def record(self, step, url=None, http_status=None, yielded=None, note=""):
+        self.steps.append(
+            ResolutionStep(
+                step=step,
+                url=url,
+                http_status=http_status,
+                yielded=dict(yielded or {}),
+                note=note,
+            )
+        )
+
+    def to_list(self) -> List[dict]:
+        return [s.to_dict() for s in self.steps]
+
+    def __len__(self):
+        return len(self.steps)
+
+    def __repr__(self):
+        return f"<ResolutionChain {len(self.steps)} steps>"
+
+
+#: Fields a source may declare in ladder.json under "requires".
+IDENTIFIER_FIELDS = (
+    "doi",
+    "pmid",
+    "pmcid",
+    "openalex_id",
+    "arxiv_id",
+    "ppr_id",
+    "elsevier_pii",
+    "repository_id",
+)
+
+
+@dataclass
+class IdentifierSet:
+    """Everything phase 1 managed to learn about one record."""
+
+    doi: Optional[str] = None
+    pmid: Optional[str] = None
+    pmcid: Optional[str] = None
+    openalex_id: Optional[str] = None
+    arxiv_id: Optional[str] = None
+    ppr_id: Optional[str] = None          # Europe PMC preprint id (PPR…)
+    elsevier_pii: Optional[str] = None
+    repository_id: Optional[str] = None   # figshare/zenodo/OSF record id
+
+    # Context that is not an identifier but gates later stages.
+    publisher_prefix: Optional[str] = None   # DOI registrant prefix, e.g. "10.1016"
+    preprint_server: Optional[str] = None    # "biorxiv" | "medrxiv"
+    license: Optional[str] = None
+    #: Europe PMC availability flags (inEPMC, isOpenAccess, hasPDF, ...), exactly
+    #: what the name says. It used to double as a general scratch bag holding
+    #: memoized Crossref and Unpaywall payloads, which made the name a lie about
+    #: the contents; those live in `memo` now.
+    epmc_flags: Dict[str, Any] = field(default_factory=dict)
+
+    #: Memoized API payloads, so two consumers of the same record share one fetch.
+    #: Keyed by source name ("crossref", "unpaywall").
+    memo: Dict[str, Any] = field(default_factory=dict)
+    crossref_links: List[dict] = field(default_factory=list)
+
+    chain: ResolutionChain = field(default_factory=ResolutionChain)
+
+    def learn(self, step: str, url=None, http_status=None, note="", **values) -> Dict[str, str]:
+        """Set any identifier fields not already known, and record the attempt.
+
+        Never overwrites a value that is already set: the first resolver to find
+        an identifier wins, so a later, less trustworthy source cannot quietly
+        redirect retrieval to a different paper.
+        """
+        gained = {}
+        for key, value in values.items():
+            if value in (None, "", []):
+                continue
+            if not hasattr(self, key):
+                raise AttributeError(f"IdentifierSet has no field {key!r}")
+            if getattr(self, key):
+                continue
+            setattr(self, key, value)
+            if isinstance(value, (str, int)):
+                gained[key] = str(value)
+        self.chain.record(step, url=url, http_status=http_status, yielded=gained, note=note)
+        return gained
+
+    @property
+    def best_id(self) -> Optional[str]:
+        """The identifier a human would use to name this record."""
+        return self.doi or self.pmcid or self.pmid or self.arxiv_id
+
+    def to_dict(self) -> dict:
+        return {
+            f: getattr(self, f)
+            for f in IDENTIFIER_FIELDS
+            if getattr(self, f)
+        }

fetchpdf/retrieval/ladder.json

@@ -0,0 +1,362 @@
+{
+  "version": 1,
+  "_comment": [
+    "Tier ladder and per-source capability map. Reorder here rather than in code.",
+    "Override wholesale with FETCHPDF_LADDER=/path/to/your.json.",
+    "Tier names must match fetchpdf.retrieval.tiers.Tier members.",
+    "'requires' lists IdentifierSet fields that must be non-empty for the source",
+    "to be attempted at all -- this is what keeps quota-limited calls off records",
+    "that could never have used them.",
+    "No 'host' key on sources: rate limiting buckets on the actual URL host inside HttpClient, so a per-source host was inert config that looked like a knob. Adjust throttling in rate_limits below, keyed by real hostname."
+  ],
+  "ladders": {
+    "extraction": [
+      "T1_XML",
+      "T2_HTML",
+      "T3_SOURCE",
+      "T4_SUPPLEMENT",
+      "T5_PDF",
+      "T7_LANDING"
+    ],
+    "screening": [
+      "T1_XML",
+      "T2_HTML",
+      "T3_SOURCE",
+      "T6_PLAINTEXT",
+      "T5_PDF",
+      "T7_LANDING"
+    ]
+  },
+  "_ladders_comment": [
+    "T6 plain text is 4th for screening and ABSENT for extraction. Flattened text",
+    "keeps every number but anchors none of them to a row or column, so an",
+    "extraction model produces confident, plausible, wrong arm-to-outcome",
+    "assignments. That is worse than a missing table, so extraction refuses it",
+    "outright rather than ranking it last and degrading into it."
+  ],
+  "tier_sources": {
+    "T1_XML": [
+      "europepmc_xml",
+      "preprint_jats",
+      "pmc_efetch",
+      "elsevier_tdm",
+      "crossref_tdm"
+    ],
+    "T2_HTML": [
+      "unpaywall_html",
+      "publisher_html",
+      "arxiv_html"
+    ],
+    "T3_SOURCE": [
+      "arxiv_eprint"
+    ],
+    "T4_SUPPLEMENT": [
+      "europepmc_supplements",
+      "figshare_files",
+      "zenodo_files",
+      "osf_files"
+    ],
+    "T5_PDF": [
+      "legacy_pdf_chain",
+      "elsevier_pdf"
+    ],
+    "T6_PLAINTEXT": [
+      "core_fulltext"
+    ],
+    "T7_LANDING": [
+      "landing_page"
+    ]
+  },
+  "sources": {
+    "europepmc_xml": {
+      "callable": "epmc:fetch_fulltext_xml",
+      "requires": [
+        "pmcid"
+      ],
+      "notes": "Highest-value T1 route. Gated on search flags, but tolerates 404 anyway: ~12% of records with a PMCID are not in the EPMC OA subset."
+    },
+    "preprint_jats": {
+      "callable": "preprint:fetch_preprint_jats",
+      "requires": [
+        "doi"
+      ],
+      "notes": "bioRxiv/medRxiv JATS. Also the route for EPMC PPR records, whose availability flags say full text exists when it does not."
+    },
+    "pmc_efetch": {
+      "callable": "pmc:fetch_efetch_xml",
+      "requires": [
+        "pmcid"
+      ],
+      "notes": "FALLBACK ONLY -- same underlying PMC JATS as Europe PMC. Never stored alongside it; two copies of one source is not corroboration."
+    },
+    "elsevier_tdm": {
+      "callable": "crossref_tdm:fetch_elsevier_tdm",
+      "requires": [
+        "elsevier_pii"
+      ],
+      "notes": "Entitlement- and IP-bound. Non-entitled requests return 200 with an error payload, so the validator, not the status code, decides."
+    },
+    "crossref_tdm": {
+      "callable": "crossref_tdm:fetch_crossref_tdm",
+      "requires": [
+        "doi"
+      ],
+      "notes": "link[] with content-type text/xml AND intended-application text-mining. Crossref retired click-through tokens in Dec 2020, so 401/403 is the expected case, not an error."
+    },
+    "unpaywall_html": {
+      "callable": "publisher_html:fetch_unpaywall_fulltext",
+      "requires": [
+        "doi"
+      ],
+      "notes": "oa_locations[].url is sometimes XML or HTML full text rather than a PDF. publishedVersion preferred over acceptedVersion."
+    },
+    "publisher_html": {
+      "callable": "publisher_html:fetch_publisher_html",
+      "requires": [
+        "doi"
+      ],
+      "notes": "DOI resolution to publisher full text. At most OA publishers this HTML is generated from the same JATS as the XML endpoint."
+    },
+    "arxiv_html": {
+      "callable": "arxiv:fetch_latexml_html",
+      "requires": [
+        "arxiv_id"
+      ],
+      "notes": "Official arxiv.org/html covers ~Dec 2023 onward; ar5iv covers the back catalogue to 2007 (verified: 0704.0001 -> 200, 35 tables). Enabled because between them they cover effectively the whole corpus. Machine conversion, so the caveat is recorded in provenance."
+    },
+    "arxiv_eprint": {
+      "callable": "arxiv:fetch_eprint_source",
+      "requires": [
+        "arxiv_id"
+      ],
+      "notes": "LaTeX source tarball. arXiv tightened rate limits in Feb 2026 -- this host gets its own slow bucket."
+    },
+    "europepmc_supplements": {
+      "callable": "supplements:fetch_epmc_supplements",
+      "requires": [
+        "pmcid"
+      ],
+      "notes": "Keyed on the PMCID already resolved in phase 1, so cheaper than enumerating repositories."
+    },
+    "figshare_files": {
+      "callable": "supplements:fetch_figshare_files",
+      "requires": [
+        "doi"
+      ]
+    },
+    "zenodo_files": {
+      "callable": "supplements:fetch_zenodo_files",
+      "requires": [
+        "doi"
+      ]
+    },
+    "osf_files": {
+      "callable": "supplements:fetch_osf_files",
+      "requires": [
+        "doi"
+      ]
+    },
+    "legacy_pdf_chain": {
+      "callable": "legacy_pdf:fetch_via_legacy_chain",
+      "requires": [
+        "doi"
+      ],
+      "notes": "One call into the existing chain in fetch_pdf_from_doi. Preserves its retries, backoff and Playwright paths by construction rather than by duplication."
+    },
+    "elsevier_pdf": {
+      "callable": "crossref_tdm:fetch_elsevier_pdf",
+      "requires": [
+        "elsevier_pii"
+      ],
+      "notes": "Same endpoint as elsevier_tdm, asked with Accept: application/pdf. Entitlement is per-representation at Elsevier, so a record whose XML view is coredata-only or a 400-char corrigendum often still serves the full typeset PDF. Ordered after legacy_pdf_chain because the chain's routes cost no Elsevier quota."
+    },
+    "core_fulltext": {
+      "callable": "core_text:fetch_core_fulltext",
+      "requires": [
+        "doi"
+      ],
+      "notes": "Always stamped tables_unavailable=true. Refuses to return anything when target_task is extraction."
+    },
+    "landing_page": {
+      "callable": "landing:fetch_landing_page",
+      "requires": [
+        "doi"
+      ],
+      "notes": "Last resort. Recorded so a record that produced nothing usable is distinguishable from one never attempted."
+    }
+  },
+  "rate_limits": {
+    "_comment": "Per-host, never global: configured limits differ by more than an order of magnitude across these APIs. rps_keyed applies when key_env is set in the environment.",
+    "eutils.ncbi.nlm.nih.gov": {
+      "rps": 3,
+      "rps_keyed": 10,
+      "key_env": "ENTREZ_EUTILS_API_KEY",
+      "burst": 3
+    },
+    "pmc.ncbi.nlm.nih.gov": {
+      "rps": 3,
+      "rps_keyed": 10,
+      "key_env": "ENTREZ_EUTILS_API_KEY",
+      "burst": 3
+    },
+    "www.ncbi.nlm.nih.gov": {
+      "rps": 3,
+      "rps_keyed": 10,
+      "key_env": "ENTREZ_EUTILS_API_KEY",
+      "burst": 3
+    },
+    "www.ebi.ac.uk": {
+      "rps": 5,
+      "burst": 5
+    },
+    "api.crossref.org": {
+      "rps": 10,
+      "burst": 10
+    },
+    "api.semanticscholar.org": {
+      "rps": 1,
+      "rps_keyed": 10,
+      "key_env": "SEMANTIC_SCHOLAR_API_KEY",
+      "burst": 1
+    },
+    "api.unpaywall.org": {
+      "rps": 5,
+      "burst": 5
+    },
+    "api.openalex.org": {
+      "rps": 5,
+      "burst": 5
+    },
+    "api.elsevier.com": {
+      "rps": 2,
+      "burst": 2
+    },
+    "api.datacite.org": {
+      "rps": 3,
+      "burst": 3
+    },
+    "api.scholexplorer.openaire.eu": {
+      "rps": 3,
+      "burst": 3
+    },
+    "api.core.ac.uk": {
+      "rps": 2,
+      "burst": 2
+    },
+    "api.biorxiv.org": {
+      "rps": 2,
+      "burst": 2
+    },
+    "export.arxiv.org": {
+      "rps": 0.34,
+      "burst": 1
+    },
+    "arxiv.org": {
+      "rps": 0.34,
+      "burst": 1
+    },
+    "ar5iv.labs.arxiv.org": {
+      "rps": 0.5,
+      "burst": 1
+    },
+    "api.figshare.com": {
+      "rps": 2,
+      "burst": 2
+    },
+    "zenodo.org": {
+      "rps": 2,
+      "burst": 2
+    },
+    "api.osf.io": {
+      "rps": 2,
+      "burst": 2
+    },
+    "doi.org": {
+      "rps": 5,
+      "burst": 5
+    },
+    "_supplementary_comment": [
+      "--pull-supplementary fetches files rather than metadata, so it talks to a",
+      "second set of hosts: the CDNs and file endpoints the listing APIs point at.",
+      "Without entries here they all fall to _default, which is safe but treats a",
+      "publisher CDN and a scraped article page as equally robust. S3 is fast and",
+      "does not care; article pages and supp.apa.org are scraped one at a time."
+    ],
+    "pmc-oa-opendata.s3.amazonaws.com": {
+      "rps": 10,
+      "burst": 10
+    },
+    "ndownloader.figshare.com": {
+      "rps": 2,
+      "burst": 2
+    },
+    "files.osf.io": {
+      "rps": 2,
+      "burst": 2
+    },
+    "osf.io": {
+      "rps": 2,
+      "burst": 2
+    },
+    "datadryad.org": {
+      "rps": 2,
+      "burst": 2
+    },
+    "dataverse.harvard.edu": {
+      "rps": 2,
+      "burst": 2
+    },
+    "journals.plos.org": {
+      "rps": 2,
+      "burst": 2
+    },
+    "media.springernature.com": {
+      "rps": 2,
+      "burst": 2
+    },
+    "static-content.springer.com": {
+      "rps": 2,
+      "burst": 2
+    },
+    "ars.els-cdn.com": {
+      "rps": 2,
+      "burst": 2
+    },
+    "www.nature.com": {
+      "rps": 1,
+      "burst": 1
+    },
+    "link.springer.com": {
+      "rps": 1,
+      "burst": 1
+    },
+    "www.biorxiv.org": {
+      "rps": 1,
+      "burst": 1
+    },
+    "www.medrxiv.org": {
+      "rps": 1,
+      "burst": 1
+    },
+    "supp.apa.org": {
+      "rps": 1,
+      "burst": 1
+    },
+    "academic.oup.com": {
+      "rps": 1,
+      "burst": 1
+    },
+    "_default": {
+      "rps": 3,
+      "burst": 3
+    }
+  },
+  "thresholds": {
+    "_comment": "Character counts are of extracted body text, not raw bytes.",
+    "t1_min_chars": 5000,
+    "t2_min_chars": 5000,
+    "abstract_stub_chars": 2500,
+    "min_populated_td": 1,
+    "idconv_chunk_size": 200
+  }
+}

fetchpdf/retrieval/linked_artifacts.py

@@ -0,0 +1,144 @@
+"""The record of what datasets and code the citation graph links to this paper.
+
+The supplement providers download files; this sidecar records *links* -- every
+publication->dataset and publication->software relation the link services
+returned, whether or not it resolved to files. The two facts are different and
+both matter downstream: a Dryad DOI with no enumerator, a trial registration,
+and an "xgboost software on GitHub" tool citation all belong here even though
+none of them lands on disk.
+
+Empty results are recorded explicitly. On the corpora this was measured against
+(2026-08), most papers have zero links -- ~30% of a recent biomedical batch had
+any ScholeXplorer link, and an older clinical-trial corpus had none at all --
+so "queried, nothing found" is the common case, and it must stay
+distinguishable from "never queried".
+
+The classification vocabulary, coarsest that survives contact with real links:
+
+  * owned          -- deposited as the paper's own material (IsSupplementTo
+                      and friends), or reached through a data DOI.
+  * related        -- a dataset/software record linked to the paper without a
+                      supplement-grade relation. Most Scholix "cites" links.
+  * tool_citation  -- a software record that is somebody's tool, not this
+                      paper's code. OpenAIRE mints these for every R package a
+                      paper cites; they dominate the software links.
+  * registry       -- a registration, not an artifact: ClinicalTrials.gov and
+                      the like. Never downloadable, always worth recording.
+
+Providers append to ctx.scratch as they run; pull_for_record writes the
+sidecar once, after the manifest. No scratch entry, no sidecar -- a run with a
+restricted provider list must not write a file claiming services were asked
+when they were not.
+"""
+
+import json
+import os
+import tempfile
+import time
+from typing import Optional
+
+_SCHEMA_VERSION = 1
+
+#: The sidecar lands beside the manifest, keyed on the same stem.
+SIDECAR_SUFFIX = "_linked_artifacts.json"
+
+#: Where providers accumulate their queries and links between enumeration and
+#: the write. One key, holding {"queried": [...], "links": [...]}.
+SCRATCH_KEY = "linked_artifacts"
+
+CLASS_OWNED = "owned"
+CLASS_RELATED = "related"
+CLASS_TOOL_CITATION = "tool_citation"
+CLASS_REGISTRY = "registry"
+
+
+def _bucket(ctx) -> dict:
+    return ctx.scratch.setdefault(SCRATCH_KEY, {"queried": [], "links": []})
+
+
+def record_query(ctx, service: str, url: str, status: str, detail: str = "") -> None:
+    """One service asked one question. `url` must already be redacted."""
+    _bucket(ctx)["queried"].append({
+        "service": service,
+        "url": url,
+        "status": status,
+        "detail": detail,
+        "retrieved_at": _iso(time.time()),
+    })
+
+
+def record_link(ctx, *, service: str, target_pid: Optional[str], target_type: str,
+                relation: str, classified: str, title: str = "",
+                publisher: str = "", provider_name: str = "",
+                harvest_date: str = "", routed: bool = False) -> None:
+    """One link the service asserted. `routed` means a repository enumerator
+    was handed the target to resolve into files."""
+    _bucket(ctx)["links"].append({
+        "service": service,
+        "target_pid": target_pid,
+        "target_type": target_type,
+        "relation": relation,
+        "classified": classified,
+        "title": (title or "")[:300],
+        "publisher": (publisher or "")[:100],
+        "provider_name": provider_name,
+        "harvest_date": harvest_date,
+        "routed_to_download": routed,
+    })
+
+
+def sidecar_path_for(stem: str) -> str:
+    return stem + SIDECAR_SUFFIX
+
+
+def write_linked_sidecar(stem: str, ids, ctx, verbose: bool = False) -> Optional[str]:
+    """Write the sidecar atomically. Never raises into the supplementary pass.
+
+    Returns the path written, or None -- both when writing failed and when
+    there was nothing to write because no link service ran.
+    """
+    bucket = ctx.scratch.get(SCRATCH_KEY)
+    if not bucket:
+        return None
+
+    links = bucket["links"]
+    record = {
+        "schema_version": _SCHEMA_VERSION,
+        "identifier": str(getattr(ids, "doi", "") or getattr(ids, "best_id", "") or ""),
+        "identifiers_resolved": ids.to_dict() if hasattr(ids, "to_dict") else {},
+        "stem": os.path.basename(stem),
+        "completed_at": _iso(time.time()),
+        "queried": bucket["queried"],
+        "links": links,
+        "counts": {
+            "links": len(links),
+            "routed_to_download": sum(1 for l in links if l.get("routed_to_download")),
+        },
+    }
+
+    path = sidecar_path_for(stem)
+    directory = os.path.dirname(os.path.abspath(path)) or "."
+    try:
+        os.makedirs(directory, exist_ok=True)
+        handle, temp_path = tempfile.mkstemp(prefix=".fetchpdf-la-", dir=directory)
+        try:
+            with os.fdopen(handle, "w", encoding="utf-8") as f:
+                json.dump(record, f, indent=2, ensure_ascii=False)
+            os.replace(temp_path, path)
+        except BaseException:
+            try:
+                os.unlink(temp_path)
+            except OSError:
+                pass
+            raise
+    except Exception as e:
+        if verbose:
+            print(f"    ⚠️  could not write {os.path.basename(path)}: {e}")
+        return None
+    return path
+
+
+def _iso(timestamp: float) -> str:
+    from datetime import datetime, timezone
+
+    return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")