fetchpdf 0.1.0__py3-none-any.whl

fetchpdf/retrieval/sources/crossref_tdm.py

@@ -0,0 +1,193 @@
+"""Crossref text-and-data-mining links, and the Elsevier endpoint most of them
+point at.
+
+Three things the obvious implementation gets wrong:
+
+1. The content-type for XML is "text/xml", not "application/xml". Verified on
+   10.1016/S0140-6736(20)30183-5, 10.1016/j.jclinepi.2021.02.003 and
+   10.1016/j.ijnurstu.2019.103466 -- all "text/xml".
+
+2. intended-application matters. "similarity-checking" links feed iThenticate
+   and are usually landing pages: 10.1371/journal.pone.0000308 exposes exactly
+   one link, content-type "unspecified", pointing at dx.plos.org. Only
+   "text-mining" links are candidates.
+
+3. There is no Crossref token to obtain. Crossref retired the click-through
+   token service in December 2020; entitlement is entirely publisher-side now.
+   So 401/403 is the expected outcome on a subscription title, not an error
+   condition, and it must never abort a run.
+
+For Elsevier the Crossref link already carries the PII inline, which is why
+Scopus is not needed to resolve one. The API key is appended here because the
+link Crossref publishes does not include it.
+"""
+
+import re
+from typing import Optional
+
+from ..._env import ELSEVIER_TDM_API_KEY
+from ..._http import elsevier_first_page_only
+from ..artifact import Artifact
+from ..tiers import Tier
+
+_XML_CONTENT_TYPES = {"text/xml", "application/xml"}
+_TEXT_MINING = "text-mining"
+_ELSEVIER_HOSTS = ("api.elsevier.com",)
+
+ELSEVIER_ARTICLE = "https://api.elsevier.com/content/article/PII:{pii}"
+
+#: Elsevier answers a non-entitled request with 200 and a coredata-only payload.
+#: Cheap pre-filter; the T1 validator is what actually decides.
+_ENTITLEMENT_SIGNATURES = (
+    b"<service-error",
+    b"AUTHENTICATION_ERROR",
+    b"APIKEY_INVALID",
+    b"RESOURCE_NOT_FOUND",
+)
+
+
+def fetch_elsevier_tdm(ids, ctx) -> Optional[Artifact]:
+    """Elsevier full-text XML by PII."""
+    if not ids.elsevier_pii:
+        return None
+    if not ELSEVIER_TDM_API_KEY:
+        ctx.log("    Elsevier TDM: no ELSEVIER_TDM_API_KEY")
+        return None
+
+    url = ELSEVIER_ARTICLE.format(pii=ids.elsevier_pii)
+    response = ctx.http.get(
+        url,
+        params={"httpAccept": "text/xml", "apiKey": ELSEVIER_TDM_API_KEY},
+        timeout=60,
+    )
+    if response.status in (401, 403):
+        ctx.log(f"    Elsevier TDM: not entitled (HTTP {response.status})")
+        return None
+    if not response.ok or not response.content:
+        ctx.log(f"    Elsevier TDM: HTTP {response.status}")
+        return None
+    if any(sig in response.content[:4096] for sig in _ENTITLEMENT_SIGNATURES):
+        ctx.log("    Elsevier TDM: 200 with an error payload (not entitled)")
+        return None
+
+    return Artifact(
+        content=response.content,
+        tier=Tier.T1_XML,
+        source="elsevier_tdm",
+        url=response.url,
+        http_status=response.status,
+        served_content_type=response.content_type,
+        identifier_used=ids.elsevier_pii,
+        license=ids.license,
+    )
+
+
+def fetch_elsevier_pdf(ids, ctx) -> Optional[Artifact]:
+    """Elsevier full-text PDF by PII -- the same endpoint, a different Accept.
+
+    Entitlement at Elsevier is negotiated per representation, not per article:
+    the XML view of a record can come back as coredata-only metadata (no
+    <body>, or a 400-character corrigendum notice) while `Accept:
+    application/pdf` on the *same* PII returns the complete typeset article.
+    Verified across four records that the XML route rejected -- all four
+    returned real PDFs of 237-446 KB.
+
+    So this is not a retry of fetch_elsevier_tdm, it is a different question
+    asked of the same URL, and it belongs at T5 because what comes back is a
+    PDF. Registered after legacy_pdf_chain in ladder.json: the chain's own
+    routes are free, and this spends Elsevier quota.
+    """
+    if not ids.elsevier_pii:
+        return None
+    if not ELSEVIER_TDM_API_KEY:
+        return None
+
+    url = ELSEVIER_ARTICLE.format(pii=ids.elsevier_pii)
+    response = ctx.http.get(
+        url,
+        headers={"X-ELS-APIKey": ELSEVIER_TDM_API_KEY,
+                 "Accept": "application/pdf"},
+        timeout=90,
+    )
+    if response.status in (401, 403):
+        ctx.log(f"    Elsevier PDF: not entitled (HTTP {response.status})")
+        return None
+    if not response.ok or not response.content:
+        ctx.log(f"    Elsevier PDF: HTTP {response.status}")
+        return None
+    # A non-entitled PDF request answers 200 with an XML error payload, so the
+    # magic bytes -- not the status -- are what decide. The engine's T5
+    # validator checks this too; failing here keeps a junk artifact from
+    # occupying the PDF goal in the first place.
+    if not response.content.startswith(b"%PDF"):
+        ctx.log("    Elsevier PDF: 200 but not a PDF (not entitled)")
+        return None
+    # Partial entitlement is a REAL one-page PDF, not an error payload: the
+    # magic bytes and the T5 validator both pass on it. The X-ELS-Status
+    # warning header is the only signal, so it must be read here where the
+    # response headers still exist -- an Artifact does not carry them.
+    if elsevier_first_page_only(response.headers):
+        ctx.log("    Elsevier PDF: first-page preview only (not entitled to full PDF)")
+        return None
+
+    return Artifact(
+        content=response.content,
+        tier=Tier.T5_PDF,
+        source="elsevier_pdf",
+        url=response.url,
+        http_status=response.status,
+        served_content_type=response.content_type,
+        identifier_used=ids.elsevier_pii,
+        license=ids.license,
+    )
+
+
+def fetch_crossref_tdm(ids, ctx) -> Optional[Artifact]:
+    """Any other publisher's TDM XML, as advertised in Crossref's link[]."""
+    if not ids.doi:
+        return None
+
+    # Lazily resolve Crossref: a record that short-circuited on a PMCID never
+    # got here, and never paid for this call.
+    ctx.resolver.crossref(ids)
+    candidates = [
+        link for link in (ids.crossref_links or [])
+        if (link.get("content-type") or "").lower() in _XML_CONTENT_TYPES
+        and (link.get("intended-application") or "").lower() == _TEXT_MINING
+    ]
+    if not candidates:
+        return None
+
+    # Version of record before accepted manuscript, when the publisher says.
+    candidates.sort(key=lambda link: 0 if (link.get("content-version") == "vor") else 1)
+
+    for link in candidates:
+        url = link.get("URL")
+        if not url:
+            continue
+        if any(host in url for host in _ELSEVIER_HOSTS):
+            # Elsevier has its own entry point that knows about the API key.
+            m = re.search(r"PII:([A-Z0-9]+)", url, re.IGNORECASE)
+            if m and not ids.elsevier_pii:
+                ids.learn("crossref.link", url=url, elsevier_pii=m.group(1))
+            continue
+
+        response = ctx.http.get(url, timeout=45)
+        if response.status in (401, 403):
+            ctx.log(f"    Crossref TDM: publisher requires entitlement (HTTP {response.status})")
+            continue
+        if not response.ok or not response.content:
+            ctx.log(f"    Crossref TDM: HTTP {response.status}")
+            continue
+        return Artifact(
+            content=response.content,
+            tier=Tier.T1_XML,
+            source="crossref_tdm",
+            url=response.url,
+            http_status=response.status,
+            served_content_type=response.content_type,
+            identifier_used=ids.doi,
+            license=ids.license,
+            extra={"content_version": link.get("content-version")},
+        )
+    return None

fetchpdf/retrieval/sources/epmc.py

@@ -0,0 +1,133 @@
+"""Europe PMC: the highest-value T1 route, and the one trap in it.
+
+Measured on a 1023-DOI clinical corpus: 48.7% of records resolve to a PMCID, and
+82.5% of those yield a JATS document that passes the T1 gate. That is ~40% of a
+batch served by one free, unauthenticated endpoint.
+
+The trap is preprints. Europe PMC indexes them as PPR records with availability
+flags identical to an OA journal article:
+
+    10.1101/2020.01.30.927871 -> id=PPR110986 source=PPR inEPMC=Y
+                                 isOpenAccess=Y hasPDF=Y
+                                 fullTextIdList={'fullTextId': ['PPR110986']}
+
+...and /PPR110986/fullTextXML returns 404. So do /supplementaryFiles and
+/textMinedTerms. Reproduced on PPR118691. The flags do not predict full text for
+preprints, so PPR records are handed to the bioRxiv/medRxiv source instead --
+and the PPR id is stashed on the IdentifierSet so that source knows to run.
+
+The flags are still worth fetching: they gate the fullTextXML call for journal
+articles and carry the license. But the fetch also tolerates a 404 on a record
+the flags called available, because ~12% of PMCID records are indexed in the ID
+Converter without being in the Europe PMC OA subset.
+"""
+
+from typing import Optional
+
+from ..artifact import Artifact
+from ..tiers import Tier
+
+REST = "https://www.ebi.ac.uk/europepmc/webservices/rest"
+
+
+def search_flags(ids, ctx) -> dict:
+    """Fetch and memoize the Europe PMC record for this DOI/PMCID.
+
+    Cheap, and it answers three questions at once: is the full text here, is it
+    open access, and is this a preprint pretending otherwise.
+    """
+    if ids.epmc_flags.get("_searched"):
+        return ids.epmc_flags
+
+    query = None
+    if ids.doi:
+        query = 'DOI:"{}"'.format(ids.doi)
+    elif ids.pmcid:
+        query = "PMCID:{}".format(ids.pmcid)
+    if not query:
+        return ids.epmc_flags
+
+    response = ctx.http.get(
+        REST + "/search",
+        params={"query": query, "format": "json", "resultType": "core"},
+        timeout=25,
+    )
+    ids.epmc_flags["_searched"] = True
+    if not response.ok:
+        ids.chain.record("europepmc.search", url=REST + "/search", http_status=response.status)
+        return ids.epmc_flags
+
+    try:
+        results = (response.json() or {}).get("resultList", {}).get("result", [])
+    except ValueError:
+        results = []
+    if not results:
+        ids.chain.record(
+            "europepmc.search", url=REST + "/search", http_status=response.status,
+            note="no Europe PMC record",
+        )
+        return ids.epmc_flags
+
+    record = results[0]
+    for key in ("inEPMC", "inPMC", "isOpenAccess", "hasPDF", "hasSuppl",
+                "hasTextMinedTerms", "source", "id"):
+        if record.get(key) is not None:
+            ids.epmc_flags[key] = record[key]
+    ids.epmc_flags["fullTextIdList"] = record.get("fullTextIdList") or {}
+
+    values = {}
+    if record.get("pmcid"):
+        values["pmcid"] = record["pmcid"]
+    if record.get("pmid"):
+        values["pmid"] = str(record["pmid"])
+    if record.get("license"):
+        values["license"] = record["license"]
+    # A PPR id is the signal that this is a preprint; the preprint source uses it.
+    if str(record.get("source") or "").upper() == "PPR" and record.get("id"):
+        values["ppr_id"] = record["id"]
+    ids.learn(
+        "europepmc.search",
+        url=REST + "/search",
+        http_status=response.status,
+        note="availability flags",
+        **values
+    )
+    return ids.epmc_flags
+
+
+def fetch_fulltext_xml(ids, ctx) -> Optional[Artifact]:
+    """JATS full text for a PMC record."""
+    flags = search_flags(ids, ctx)
+
+    if str(flags.get("source") or "").upper() == "PPR":
+        ctx.log("    Europe PMC: preprint (PPR) record -- flags do not imply full text here")
+        return None
+
+    if not ids.pmcid:
+        return None
+
+    # Gate on the flags rather than blind-fetching, but do not treat them as
+    # authoritative: fetch anyway when they are simply absent.
+    if flags and flags.get("inEPMC") == "N" and flags.get("inPMC") == "N":
+        ctx.log(f"    Europe PMC: {ids.pmcid} flagged as not in EPMC/PMC")
+        return None
+
+    url = "{}/{}/fullTextXML".format(REST, ids.pmcid)
+    response = ctx.http.get(url, timeout=45)
+    if not response.ok or not response.content:
+        # Expected for ~12% of PMCID records: indexed, but not in the OA subset.
+        ctx.log(f"    Europe PMC: {ids.pmcid} fullTextXML HTTP {response.status}")
+        return None
+
+    ctx.pmc_xml_already_taken = True
+    return Artifact(
+        content=response.content,
+        tier=Tier.T1_XML,
+        source="europepmc_xml",
+        url=response.url,
+        http_status=response.status,
+        served_content_type=response.content_type,
+        identifier_used=ids.pmcid,
+        license=flags.get("license") or ids.license,
+        extra={"epmc_flags": {k: v for k, v in flags.items() if not k.startswith("_")}},
+    )

fetchpdf/retrieval/sources/landing.py

@@ -0,0 +1,34 @@
+"""T7: the landing page. Last rung.
+
+Kept as a real source rather than an implicit "nothing found" so that a record
+which produced only a landing page is distinguishable from one that was never
+attempted. That difference matters when re-running with --upgrade-existing:
+the first is a known dead end, the second is work still to do.
+"""
+
+from typing import Optional
+
+from ..artifact import Artifact
+from ..tiers import Tier
+
+DOI_RESOLVER = "https://doi.org/{doi}"
+
+
+def fetch_landing_page(ids, ctx) -> Optional[Artifact]:
+    if not ids.doi:
+        return None
+    response = ctx.http.get(DOI_RESOLVER.format(doi=ids.doi), timeout=30, polite=False)
+    if not response.ok or not response.content:
+        return None
+    return Artifact(
+        content=response.content,
+        tier=Tier.T7_LANDING,
+        source="landing_page",
+        url=response.url,
+        http_status=response.status,
+        served_content_type=response.content_type,
+        identifier_used=ids.doi,
+        license=ids.license,
+        extension=".landing.html",
+        tables_unavailable=True,
+    )

fetchpdf/retrieval/sources/legacy_pdf.py

@@ -0,0 +1,99 @@
+"""T5: the existing PDF chain, called as a single source.
+
+Deliberately not reimplemented. The chain in fetchpdf is ~1000 lines
+of accumulated publisher-specific knowledge -- Wiley rendered PDFs, the Crossref
+chooser page, PsychArchives bitstreams, the eScholarship LinkOut route, session
+cookies, SSL fallbacks and the Playwright paths. Forking it to slot into a tier would duplicate every one of
+those behaviours and let the two copies drift.
+
+So T5 is one call back into it with the tiered flags off. Its retries, backoff,
+rate limiting and source tracking are preserved by construction rather than by
+being carefully re-derived here.
+
+It writes to a temporary path in the destination directory rather than to the
+caller's save_path, because the engine has not decided to accept anything yet --
+a T5 artifact that fails classification must leave no file behind.
+"""
+
+import os
+import tempfile
+from typing import Optional
+
+from ..artifact import Artifact
+from ..tiers import Tier
+
+
+def fetch_via_legacy_chain(ids, ctx) -> Optional[Artifact]:
+    if not ids.doi:
+        return None
+
+    # Imported at call time: fetchpdf imports the engine, so a
+    # module-scope import here would be a cycle.
+    #
+    # The CHAIN, not the `fetch_pdf` wrapper. The wrapper falls back to a
+    # structured-full-text walk when it cannot produce a PDF -- correct for a
+    # caller who asked for a paper, wrong here, where the tiered engine has
+    # ALREADY walked and failed T1 and T2 before descending to this rung. Going
+    # through the wrapper re-ran that walk verbatim for every hard record, with
+    # `resolver=None`, so it also discarded the primed batch resolver and built
+    # a fresh limiter, client and cache per record -- then announced it had
+    # saved structured text to a temp path the caller never sees.
+    from ...fetchpdf import _fetch_pdf_chain as fetch_pdf
+    from ...fetchpdf import download_url_for
+
+    directory = os.path.dirname(os.path.abspath(ctx.save_path)) or "."
+    os.makedirs(directory, exist_ok=True)
+    handle, temp_path = tempfile.mkstemp(prefix=".fetchpdf-t5-", suffix=".pdf", dir=directory)
+    os.close(handle)
+    # The chain's own skip-if-exists check would see the empty temp file and
+    # return immediately, so it has to be gone before the call.
+    _unlink(temp_path)
+
+    source_out = [None]
+    written = None
+    try:
+        written = fetch_pdf(
+            ids.doi,
+            temp_path,
+            email=ctx.email,
+            verbose=ctx.verbose,
+            delay=ctx.delay,
+            use_playwright=ctx.use_playwright,
+            _source_out=source_out,
+        )
+        if not written or not os.path.exists(written):
+            return None
+        with open(written, "rb") as f:
+            content = f.read()
+        if not content:
+            return None
+
+        # The chain's own Elsevier XML fallback writes .xml, not .pdf. That is a
+        # T1 artifact arriving through the T5 door; the engine's classifier will
+        # re-tier it, so it is simply passed along as found.
+        return Artifact(
+            content=content,
+            tier=Tier.T5_PDF,
+            source="legacy_pdf_chain",
+            # The URL the bytes actually came from, recorded by try_download.
+            # This was "" until now, which is why a wrong artifact on disk could
+            # not say where it came from.
+            url=download_url_for(written) or download_url_for(temp_path),
+            http_status=200,
+            served_content_type="application/pdf",
+            identifier_used=ids.doi,
+            license=ids.license,
+            extra={"legacy_source": source_out[0]},
+        )
+    finally:
+        for path in {temp_path, written}:
+            if path:
+                _unlink(path)
+
+
+def _unlink(path: str) -> None:
+    try:
+        if os.path.exists(path):
+            os.unlink(path)
+    except OSError:
+        pass

fetchpdf/retrieval/sources/pmc.py

@@ -0,0 +1,59 @@
+"""NCBI E-utilities efetch against db=pmc. Fallback only.
+
+Same underlying PMC JATS as Europe PMC serves, so it runs only when the Europe
+PMC route did not produce an artifact. Storing both copies would not be
+corroboration -- it would manufacture false agreement between two views of one
+source, which is worse than having one.
+
+This endpoint is also the cleanest example of why HTTP 200 cannot be trusted.
+For a non-OA record it returns a complete, well-formed document -- journal
+metadata, title, authors, the lot -- with no <body> element, and the reason
+stated in an XML comment that every parser silently discards:
+
+    <!--The publisher of this article does not allow downloading of the full
+        text in XML form.-->
+
+Verified on PMC3390974 and PMC2148499. The T1 validator catches it against the
+raw bytes; the check lives there rather than here so every XML source gets it.
+
+Scope note on the August 2026 NCBI change: it removes legacy *bulk dataset
+files* from the FTP and Cloud services. The OA Web Service and E-utilities are
+not in scope, and this package touches no FTP or S3 path, so nothing here needs
+to migrate.
+"""
+
+from typing import Optional
+
+from ..artifact import Artifact
+from ..tiers import Tier
+
+EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
+
+
+def fetch_efetch_xml(ids, ctx) -> Optional[Artifact]:
+    if not ids.pmcid:
+        return None
+    if ctx.pmc_xml_already_taken:
+        ctx.log("    NCBI efetch: skipped, Europe PMC already served this PMC record")
+        return None
+
+    numeric_id = str(ids.pmcid).upper().replace("PMC", "")
+    response = ctx.http.get(
+        EFETCH,
+        params={"db": "pmc", "id": numeric_id, "retmode": "xml"},
+        timeout=45,
+    )
+    if not response.ok or not response.content:
+        ctx.log(f"    NCBI efetch: HTTP {response.status} for {ids.pmcid}")
+        return None
+
+    return Artifact(
+        content=response.content,
+        tier=Tier.T1_XML,
+        source="pmc_efetch",
+        url=response.url,
+        http_status=response.status,
+        served_content_type=response.content_type,
+        identifier_used=ids.pmcid,
+        license=ids.license,
+    )

fetchpdf/retrieval/sources/preprint.py

@@ -0,0 +1,101 @@
+"""bioRxiv / medRxiv JATS.
+
+Their details API exposes a `jatsxml` field pointing at the full JATS document:
+
+    api.biorxiv.org/details/biorxiv/10.1101/2020.01.30.927871
+      -> jatsxml: https://www.biorxiv.org/content/early/2020/01/31/....source.xml
+
+The trap is the server segment. A medRxiv DOI queried against /details/biorxiv/
+returns HTTP 200 with {"collection": [], "messages": [{"status": "no posts
+found"}]} -- a miss that looks exactly like a successful empty answer. Querying
+the wrong server for every medRxiv record would silently report "no full text"
+for the entire preprint half of a corpus, so both servers are tried and the
+first non-empty collection wins.
+
+This is also where Europe PMC PPR records land: EPMC indexes preprints with
+inEPMC=Y and a populated fullTextIdList but 404s on fullTextXML, so the flags
+cannot be used and the publisher's own API is the real route.
+"""
+
+from typing import Optional
+
+from ..artifact import Artifact
+from ..tiers import Tier
+
+DETAILS = "https://api.biorxiv.org/details/{server}/{doi}"
+_SERVERS = ("biorxiv", "medrxiv")
+_PREPRINT_PREFIX = "10.1101/"
+
+
+def fetch_preprint_jats(ids, ctx) -> Optional[Artifact]:
+    doi = (ids.doi or "").lower()
+    is_preprint = doi.startswith(_PREPRINT_PREFIX) or bool(ids.ppr_id)
+    if not is_preprint:
+        return None
+
+    record = None
+    for server in _order_servers(ids.preprint_server):
+        record = _latest_version(ctx, server, ids.doi)
+        if record:
+            ids.learn(
+                "biorxiv.details",
+                url=DETAILS.format(server=server, doi=ids.doi),
+                note="preprint server identified",
+                preprint_server=server,
+                license=record.get("license"),
+            )
+            break
+
+    if not record:
+        ctx.log("    preprint: no record on either bioRxiv or medRxiv")
+        return None
+
+    jats_url = record.get("jatsxml")
+    if not jats_url:
+        ctx.log("    preprint: record has no jatsxml field")
+        return None
+
+    response = ctx.http.get(jats_url, timeout=45)
+    if not response.ok or not response.content:
+        ctx.log(f"    preprint: jatsxml HTTP {response.status}")
+        return None
+
+    return Artifact(
+        content=response.content,
+        tier=Tier.T1_XML,
+        source="preprint_jats",
+        url=response.url,
+        http_status=response.status,
+        served_content_type=response.content_type,
+        identifier_used=ids.doi,
+        license=record.get("license"),
+        extra={"version": record.get("version"), "server": record.get("server")},
+    )
+
+
+def _order_servers(known: Optional[str]):
+    """Try the known server first; 'unknown' means the DOI prefix told us nothing."""
+    if known in _SERVERS:
+        return (known,) + tuple(s for s in _SERVERS if s != known)
+    return _SERVERS
+
+
+def _latest_version(ctx, server: str, doi: str) -> Optional[dict]:
+    """The highest-version record, or None for the empty-collection miss."""
+    response = ctx.http.get(DETAILS.format(server=server, doi=doi), timeout=25)
+    if not response.ok:
+        return None
+    try:
+        collection = (response.json() or {}).get("collection") or []
+    except ValueError:
+        return None
+    if not collection:
+        return None
+    return max(collection, key=lambda r: _version_of(r))
+
+
+def _version_of(record: dict) -> int:
+    try:
+        return int(record.get("version") or 0)
+    except (TypeError, ValueError):
+        return 0