fetchpdf 0.1.0__py3-none-any.whl

fetchpdf/retrieval/pdf_text.py

@@ -0,0 +1,242 @@
+"""Text out of a PDF, for the code that has to read the paper itself.
+
+Two consumers, one seam: `fulltext_scan` looks for deposit URLs in the prose,
+and the retrieval agent needs the same prose as its brief. Both were blind to
+PDF-only records -- measured 2026-08-22 across four corpora, **606 papers have
+a PDF and no XML or HTML at all** (cerebrolysin 212, masliah 369, nina_mazar
+25). Those were never scanned, and a record that was never scanned was being
+reported as one where nothing was found.
+
+OPTIONAL BY CONTRACT, AND STILL IS. Neither engine raises here, and a missing
+one degrades to "no text" with a named reason -- never an exception, never a
+failed record. `engine_status()` says which of the three states holds so a
+caller can tell "this machine has no PDF reader" from "this PDF has no text
+layer", which are different facts about different things.
+
+What changed is the POLICY LAYER above it, not this contract. A third consumer,
+`pdf_identity`, uses this module to prove a retrieved PDF is the paper that was
+asked for; with no engine it cannot prove anything, and under reject-always
+that would silently discard every PDF in a run. So `pypdf` moved into the base
+dependencies and the CLI refuses to start without an engine -- a RUN-level
+error answered once at startup, not a per-record one answered 5,000 times. The
+module keeps degrading; the caller owns the policy. Do not make this raise.
+
+PyMuPDF first, pypdf second. PyMuPDF is faster and keeps reading order better;
+pypdf is the fallback already used by `corresponding.from_pdf`, so a machine
+set up for that keeps working here.
+"""
+
+import os
+from typing import Optional, Tuple
+
+#: Engines, in preference order. Names are what `engine_status` reports.
+ENGINE_PYMUPDF = "pymupdf"
+ENGINE_PYPDF = "pypdf"
+
+#: Reported when neither engine imports. The message names the install, because
+#: the alternative is a silent zero-yield scan that reads like a clean corpus.
+MISSING_DEP_MSG = ("no PDF text engine, so no retrieved PDF can be checked "
+                   "against the record it was fetched for; install pypdf, or "
+                   "fetchpdf[text] for the faster PyMuPDF reader")
+
+#: A page cap for callers that only need the front or back matter. None reads
+#: the whole document, which is what the deposit scan wants -- a data
+#: availability statement lives in the back matter, not on page one.
+DEFAULT_MAX_PAGES = None
+
+#: Below this many characters we treat the read as "no text layer" rather than
+#: as text. A scanned page yields a handful of stray glyphs, not prose, and
+#: calling that a successful read invites the caller to conclude the paper
+#: named no deposit when nothing was ever legible.
+MIN_USEFUL_CHARS = 500
+
+
+def _pymupdf():
+    """PyMuPDF if installed, else None. Optional extra: fetchpdf[text]."""
+    try:
+        import pymupdf
+        return pymupdf
+    except ImportError:
+        try:
+            import fitz
+            return fitz
+        except ImportError:
+            return None
+
+
+def _pypdf():
+    """pypdf if installed, else None."""
+    try:
+        from pypdf import PdfReader
+        return PdfReader
+    except ImportError:
+        return None
+
+
+def engine_status() -> Tuple[Optional[str], str]:
+    """`(engine_name, detail)`. `engine_name` is None when none is installed."""
+    if _pymupdf() is not None:
+        return ENGINE_PYMUPDF, "PyMuPDF"
+    if _pypdf() is not None:
+        return ENGINE_PYPDF, "pypdf"
+    return None, MISSING_DEP_MSG
+
+
+def pdf_text(path: str, max_pages: Optional[int] = DEFAULT_MAX_PAGES,
+             min_chars: int = MIN_USEFUL_CHARS) -> Optional[str]:
+    """The document's text, or None when it could not be read as text.
+
+    None covers every failure the same way on purpose -- no engine, an
+    unreadable file, an encrypted document, a page tree of images. The caller
+    that needs to tell those apart asks `engine_status()` first; the caller
+    that just wants prose gets "there is none" without a traceback.
+
+    `min_chars` exists because MIN_USEFUL_CHARS is calibrated for the whole-
+    document deposit scan, where a few hundred stray glyphs mean "scanned, no
+    text layer". A caller reading only the first two pages of a short paper can
+    fall under that floor with perfectly good text, and reporting that as "no
+    text layer" would, under reject-always, delete a correct PDF. Identity
+    verification passes a lower floor; the default is unchanged, so
+    `fulltext_scan` is untouched.
+    """
+    if not path or not os.path.exists(path):
+        return None
+    return _extract(path, max_pages, min_chars)
+
+
+def pdf_text_from_bytes(content: bytes,
+                        max_pages: Optional[int] = DEFAULT_MAX_PAGES,
+                        min_chars: int = MIN_USEFUL_CHARS) -> Optional[str]:
+    """`pdf_text` for a document that is still in memory.
+
+    The tiered engine judges an artifact before anything has been written to
+    disk -- deliberately, so a T5 artifact that fails validation leaves no file
+    behind -- so it has bytes where the scan has a path.
+    """
+    if not content:
+        return None
+    return _extract(content, max_pages, min_chars)
+
+
+def pdf_page_count(source) -> Optional[int]:
+    """How many pages the document has, or None if it cannot be opened.
+
+    Structural, not textual: this works on an image-only scan, which is the
+    whole reason it exists. See `pdf_identity`, where a page count is the only
+    identity evidence a scanned paper can offer.
+    """
+    module = _pymupdf()
+    if module is not None:
+        document = None
+        try:
+            document = _open_pymupdf(module, source)
+            return int(document.page_count)
+        except Exception:
+            pass
+        finally:
+            if document is not None:
+                try:
+                    document.close()
+                except Exception:
+                    pass
+
+    reader_cls = _pypdf()
+    if reader_cls is not None:
+        try:
+            return len(reader_cls(_as_pypdf_source(source)).pages)
+        except Exception:
+            return None
+    return None
+
+
+def pdf_metadata(source) -> dict:
+    """The document's Info/XMP dictionary, or {} for anything unreadable.
+
+    Accepts a path or bytes. Only the embedded `Title` is used today, as the
+    last identity signal when a PDF's text layer is too thin to carry one.
+    """
+    module = _pymupdf()
+    if module is not None:
+        document = None
+        try:
+            document = _open_pymupdf(module, source)
+            return dict(document.metadata or {})
+        except Exception:
+            pass
+        finally:
+            if document is not None:
+                try:
+                    document.close()
+                except Exception:
+                    pass
+
+    reader_cls = _pypdf()
+    if reader_cls is not None:
+        try:
+            reader = reader_cls(_as_pypdf_source(source))
+            return {str(k).lstrip("/").lower(): str(v)
+                    for k, v in (reader.metadata or {}).items()}
+        except Exception:
+            return {}
+    return {}
+
+
+def _open_pymupdf(module, source):
+    if isinstance(source, (bytes, bytearray)):
+        return module.open(stream=bytes(source), filetype="pdf")
+    return module.open(source)
+
+
+def _as_pypdf_source(source):
+    if isinstance(source, (bytes, bytearray)):
+        import io
+        return io.BytesIO(bytes(source))
+    return source
+
+
+def _extract(source, max_pages: Optional[int], min_chars: int) -> Optional[str]:
+    """PyMuPDF first, pypdf second, over either a path or bytes."""
+    module = _pymupdf()
+    if module is not None:
+        text = _read_pymupdf(module, source, max_pages, min_chars)
+        if text is not None:
+            return text
+
+    reader = _pypdf()
+    if reader is not None:
+        return _read_pypdf(reader, source, max_pages, min_chars)
+
+    return None
+
+
+def _read_pymupdf(module, source, max_pages: Optional[int],
+                  min_chars: int = MIN_USEFUL_CHARS) -> Optional[str]:
+    document = None
+    try:
+        document = _open_pymupdf(module, source)
+        pages = list(document)
+        if max_pages is not None:
+            pages = pages[:max_pages]
+        text = "\n".join(page.get_text() or "" for page in pages)
+    except Exception:
+        # A corrupt or encrypted PDF is not this module's problem to raise on;
+        # the pypdf fallback gets its turn, and then the caller gets None.
+        return None
+    finally:
+        if document is not None:
+            try:
+                document.close()
+            except Exception:
+                pass
+    return text if len(text.strip()) >= min_chars else None
+
+
+def _read_pypdf(reader_cls, source, max_pages: Optional[int],
+                min_chars: int = MIN_USEFUL_CHARS) -> Optional[str]:
+    try:
+        reader = reader_cls(_as_pypdf_source(source))
+        pages = reader.pages if max_pages is None else reader.pages[:max_pages]
+        text = "\n".join((page.extract_text() or "") for page in pages)
+    except Exception:
+        return None
+    return text if len(text.strip()) >= min_chars else None

fetchpdf/retrieval/provenance.py

@@ -0,0 +1,185 @@
+"""The per-record audit sidecar.
+
+The requirement this satisfies is not "nice logging". It is: given a number in
+an extraction output, reconstruct why it came from that file -- which source
+served it, which identifier got us there, which validation checks it passed, and
+what we tried and rejected first.
+
+That last part is why `attempts` records demotions as well as the accepted
+artifact. "We used the PDF" is not an audit trail; "the Europe PMC XML was a
+publisher denial stub and the Crossref TDM link 403'd, so we used the PDF" is.
+
+Written only when --provenance is passed. Without it no sidecar appears, so an
+existing output directory gains no new files.
+"""
+
+import json
+import os
+import tempfile
+import time
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+from .http import redact
+from ._util import iso as _iso
+
+SIDECAR_SUFFIX = ".provenance.json"
+_SCHEMA_VERSION = 2
+
+
+@dataclass
+class Attempt:
+    """One source tried, accepted or demoted."""
+
+    source: str
+    tier_attempted: str
+    url: Optional[str] = None
+    http_status: Optional[int] = None
+    served_content_type: str = ""
+    accepted: bool = False
+    outcome: str = ""                       # why it was demoted, or "accepted"
+    checks_passed: List[str] = field(default_factory=list)
+    timestamp: float = field(default_factory=time.time)
+
+    def to_dict(self) -> dict:
+        return {
+            "source": self.source,
+            "tier_attempted": self.tier_attempted,
+            "url": redact(self.url),
+            "http_status": self.http_status,
+            "served_content_type": self.served_content_type,
+            "accepted": self.accepted,
+            "outcome": self.outcome,
+            "checks_passed": list(self.checks_passed),
+            "timestamp": _iso(self.timestamp),
+        }
+
+
+class ProvenanceRecord:
+    """Accumulates everything about one record's retrieval."""
+
+    def __init__(self, identifier: str, target_task: str, flags: Optional[dict] = None):
+        self.identifier = identifier
+        self.target_task = target_task
+        self.flags = dict(flags or {})
+        self.attempts: List[Attempt] = []
+        self.started_at = time.time()
+        self.artifact: Optional[dict] = None   # the best one, for single-artifact readers
+        self.artifacts: List[dict] = []
+        self.resolution_chain: List[dict] = []
+        self.identifiers: Dict[str, str] = {}
+        self.tables: List[dict] = []
+        self.notes: List[str] = []
+
+    def attempt(self, **kwargs) -> Attempt:
+        record = Attempt(**kwargs)
+        self.attempts.append(record)
+        return record
+
+    def note(self, message: str) -> None:
+        self.notes.append(message)
+
+    def accept(self, artifact, path: str, chunks=None, canonical_tokens: int = 0) -> None:
+        """Record an accepted artifact.
+
+        Appends. A --get-xml-or-html walk accepts two (a structured copy and the
+        PDF), and an audit record for such a record has to describe both -- which
+        one a number came from is the question this file exists to answer.
+        `self.artifact` stays pointing at the first, best one for readers that
+        expect a single entry.
+        """
+        validation = artifact.validation
+        entry = {
+            "path": os.path.basename(path),
+            "source": artifact.source,
+            "tier": artifact.tier.name,
+            "tier_value": int(artifact.tier),
+            "tier_declared_by_source": (
+                artifact.declared_tier.name if artifact.declared_tier else artifact.tier.name
+            ),
+            "tier_assigned_by": "content inspection",
+            "why_this_tier": {
+                "checks_passed": list(validation.checks_passed) if validation else [],
+                "reason": validation.reason if validation else "",
+            },
+            "served_content_type": artifact.served_content_type,
+            "identifier_used": artifact.identifier_used,
+            "request_url": redact(artifact.url),
+            "http_status": artifact.http_status,
+            "retrieved_at": _iso(time.time()),
+            "content_hash": artifact.content_hash,
+            "content_bytes": len(artifact.content),
+            "license": artifact.license,
+            "table_count": validation.n_tables if validation else 0,
+            "footnote_count": validation.n_footnotes if validation else 0,
+            "populated_cell_count": validation.n_populated_cells if validation else 0,
+            "canonical_token_count": canonical_tokens,
+            "tables_unavailable": artifact.tables_unavailable,
+            "normalization_failures": list(artifact.normalization_failures),
+        }
+        self.artifacts.append(entry)
+        if self.artifact is None:
+            self.artifact = entry
+        if chunks:
+            self.tables.extend(c.to_dict() for c in chunks)
+        # Zero tables in a trial paper is a finding, not a normal outcome -- but
+        # only where we actually counted. A PDF's table count is 0 because
+        # nothing here parses PDFs, which is not the same claim at all.
+        countable = artifact.tier.name in ("T1_XML", "T2_HTML")
+        if countable and entry["table_count"] == 0 and not artifact.tables_unavailable:
+            self.note("zero tables in accepted artifact -- verify this paper has none")
+
+    def to_dict(self) -> dict:
+        return {
+            "schema_version": _SCHEMA_VERSION,
+            "identifier": self.identifier,
+            "target_task": self.target_task,
+            "flags": self.flags,
+            "started_at": _iso(self.started_at),
+            "completed_at": _iso(time.time()),
+            "identifiers_resolved": dict(self.identifiers),
+            "resolution_chain": [
+                {**step, "url": redact(step.get("url"))} for step in self.resolution_chain
+            ],
+            "attempts": [a.to_dict() for a in self.attempts],
+            "artifact": self.artifact,
+            "artifacts": list(self.artifacts),
+            "tables": self.tables,
+            "notes": list(self.notes),
+        }
+
+
+def sidecar_path_for(artifact_path: str) -> str:
+    """Sidecar sits beside the artifact, keyed on the same stem."""
+    stem = artifact_path
+    for suffix in (".source.tar.gz", ".fulltext.html", ".landing.html", ".suppl.zip"):
+        if stem.endswith(suffix):
+            return stem[: -len(suffix)] + SIDECAR_SUFFIX
+    return os.path.splitext(stem)[0] + SIDECAR_SUFFIX
+
+
+def write_sidecar(record: ProvenanceRecord, artifact_path: str, verbose: bool = False) -> Optional[str]:
+    """Write the sidecar atomically. Never raises into the retrieval path."""
+    path = sidecar_path_for(artifact_path)
+    directory = os.path.dirname(os.path.abspath(path)) or "."
+    try:
+        os.makedirs(directory, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(prefix=".fetchpdf-prov-", dir=directory)
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as f:
+                json.dump(record.to_dict(), f, indent=2, ensure_ascii=False)
+            os.replace(tmp, path)
+        except BaseException:
+            try:
+                os.unlink(tmp)
+            except OSError:
+                pass
+            raise
+    except OSError as e:
+        if verbose:
+            print(f"  could not write provenance sidecar: {e}")
+        return None
+    return path
+
+
+

fetchpdf/retrieval/ratelimit.py

@@ -0,0 +1,113 @@
+"""Per-host token buckets.
+
+Per-host and not global, because the configured limits differ by more than an
+order of magnitude across the APIs this package touches: NCBI E-utilities is
+3/s unkeyed and 10/s keyed, Semantic Scholar is 1/s unkeyed, Crossref's polite
+pool is 10/s, and arXiv now wants roughly one request every three seconds. A
+single global limiter set safe enough for arXiv would make a PMC-heavy batch
+take all day; set fast enough for Crossref it would get us 429'd off arXiv.
+
+Buckets are process-wide and lock-guarded so the ThreadPoolExecutor in
+batch_fetch_pdfs shares one budget per host rather than one per worker.
+"""
+
+import os
+import threading
+import time
+from typing import Dict, Optional
+
+
+class TokenBucket:
+    """Classic token bucket: `rate` tokens/second, capped at `burst`."""
+
+    def __init__(self, rate: float, burst: float):
+        self.rate = max(float(rate), 0.001)
+        self.burst = max(float(burst), 1.0)
+        self._tokens = self.burst
+        self._last = time.monotonic()
+        self._lock = threading.Lock()
+
+    def acquire(self, tokens: float = 1.0) -> float:
+        """Block until `tokens` are available. Returns seconds actually slept."""
+        slept = 0.0
+        while True:
+            with self._lock:
+                now = time.monotonic()
+                self._tokens = min(
+                    self.burst, self._tokens + (now - self._last) * self.rate
+                )
+                self._last = now
+                if self._tokens >= tokens:
+                    self._tokens -= tokens
+                    return slept
+                deficit = tokens - self._tokens
+                wait = deficit / self.rate
+            time.sleep(wait)
+            slept += wait
+
+    def penalize(self, seconds: float) -> None:
+        """Drain the bucket after a 429, so the next caller waits it out."""
+        with self._lock:
+            self._tokens = 0.0
+            self._last = time.monotonic() + max(0.0, seconds)
+
+
+#: The process-wide limiter. Politeness is a property of the PROCESS, not of a
+#: batch: an API's allowance does not multiply because we started a second
+#: walk. Three call sites used to build their own -- the tiered engine, the
+#: supplementary pass and the legacy chain's Crossref helper -- so a run that
+#: touched two of them had two budgets for the same host and could serve
+#: Crossref 20 requests a second against a published allowance of 10.
+#:
+#: First config wins, which is correct here because every caller passes the
+#: same `ladder.rate_limits`. Tests that want an isolated limiter construct
+#: HostRateLimiter directly.
+_SHARED_LIMITER = None
+_SHARED_LOCK = threading.Lock()
+
+
+def shared_host_limiter(config: Optional[dict] = None) -> "HostRateLimiter":
+    """The one limiter every production caller should use."""
+    global _SHARED_LIMITER
+    with _SHARED_LOCK:
+        if _SHARED_LIMITER is None:
+            _SHARED_LIMITER = HostRateLimiter(config)
+        return _SHARED_LIMITER
+
+
+class HostRateLimiter:
+    """Resolves a host to its bucket, honoring the keyed/unkeyed distinction."""
+
+    def __init__(self, config: Optional[dict] = None):
+        self._config = dict(config or {})
+        self._default = self._config.get("_default", {"rps": 3, "burst": 3})
+        self._buckets: Dict[str, TokenBucket] = {}
+        self._lock = threading.Lock()
+
+    def _spec_for(self, host: str) -> dict:
+        spec = self._config.get(host)
+        if not isinstance(spec, dict):
+            return self._default
+        return spec
+
+    def bucket(self, host: Optional[str]) -> TokenBucket:
+        key = host or "_default"
+        with self._lock:
+            bucket = self._buckets.get(key)
+            if bucket is not None:
+                return bucket
+            spec = self._spec_for(key)
+            rate = spec.get("rps", 3)
+            key_env = spec.get("key_env")
+            # The keyed allowance only applies if we actually hold the key.
+            if key_env and os.getenv(key_env) and spec.get("rps_keyed"):
+                rate = spec["rps_keyed"]
+            bucket = TokenBucket(rate=rate, burst=spec.get("burst", rate))
+            self._buckets[key] = bucket
+            return bucket
+
+    def acquire(self, host: Optional[str]) -> float:
+        return self.bucket(host).acquire()
+
+    def penalize(self, host: Optional[str], seconds: float) -> None:
+        self.bucket(host).penalize(seconds)

fetchpdf/retrieval/repository_waf.py

@@ -0,0 +1,152 @@
+"""Repository APIs behind a JavaScript bot-challenge, via a real browser tab.
+
+Harvard Dataverse sits behind AWS WAF. When the challenge is armed, *every*
+endpoint -- the dataset API, `/api/access`, `/api/info/version`, even the site
+root -- answers **HTTP 202 with an empty body** to a plain HTTP client. Measured
+2026-08-12 against 10.7910/DVN/Z8WAKW, same machine and network:
+
+    curl, no UA                             202, 0 bytes
+    curl, browser UA                        202, 2071 bytes  <- the challenge page
+    demo.dataverse.org (control)            200               <- not us, not the network
+    Playwright HEADED, land then ask API    200, 1 file       <- works
+
+202-with-no-body is the part that misleads. It is not 5xx, not 403, not a
+timeout; `response.ok` is false and the enumerator's `return []` reads as "this
+deposit has no files". A trial's own individual-level replication data was
+skipped that way while the paper was reviewed against its summary statistics
+alone, and the manifest recorded `none_found` -- a claim about the authors
+written out of somebody else's bot protection.
+
+The challenge page is recognisable: AWS WAF ships `window.gokuProps` and
+`awsWafCookieDomainList` and needs JavaScript to solve. A real browser solves it
+in a few seconds and receives a cookie; the API then answers normally *in that
+browser context*, which is why the request is issued through `page.request`
+rather than copied back out to the HTTP client.
+
+Two things are load-bearing, both learned in supplement_atypon.py:
+
+  * **Headed, not headless.** On a headless server that means xvfb. The helpers
+    are imported from supplement_atypon rather than copied -- one xvfb
+    implementation, not two that drift.
+  * **Land on the page first.** The challenge is issued for the site, not for
+    the endpoint. Navigating straight to the API URL hands the challenge to a
+    JSON client that cannot run it.
+
+Expensive -- one browser per record -- so it is the last resort, reached only
+after the plain HTTP enumerator has already failed against a host known to use
+this protection.
+"""
+
+import json
+import time
+from typing import Optional
+
+#: Hosts observed to serve a JS bot-challenge in front of their API. Keyed by
+#: host so a repository that later drops the challenge can simply be removed.
+#: The landing template gives the browser a page to solve the challenge on
+#: before the API is asked.
+WAF_HOSTS = {
+    "dataverse.harvard.edu":
+        "https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:{doi}",
+}
+
+#: Markers of an AWS WAF interstitial. Present in the 202 body; absent once the
+#: challenge is solved, which is how the wait knows it is done.
+_CHALLENGE_MARKERS = ("gokuProps", "awsWafCookieDomainList")
+
+#: A challenge that has not cleared in this long is not going to.
+_CHALLENGE_TIMEOUT_S = 25
+
+
+def host_uses_waf(url: str) -> Optional[str]:
+    """The landing-page template for `url`'s host, or None."""
+    for host, landing in WAF_HOSTS.items():
+        if host in (url or ""):
+            return landing
+    return None
+
+
+def looks_like_challenge(body: bytes, status: int) -> bool:
+    """Whether this response is a bot-challenge rather than an answer.
+
+    An empty 202 counts: that is what the WAF returns to a client it will not
+    even hand the challenge page to, and treating it as a valid empty result is
+    the whole failure this module exists to prevent.
+    """
+    if status == 202:
+        return True
+    if not body:
+        return False
+    try:
+        head = body[:4096].decode("utf-8", "ignore")
+    except Exception:
+        return False
+    return any(marker in head for marker in _CHALLENGE_MARKERS)
+
+
+def _page_content(page) -> str:
+    """page.content() while the WAF is mid-redirect raises; retry briefly."""
+    for _ in range(10):
+        try:
+            return page.content()
+        except Exception:
+            time.sleep(1)
+    return ""
+
+
+def fetch_json_through_browser(api_url: str, doi: str, ctx) -> Optional[dict]:
+    """Solve the host's challenge in a real browser, then GET `api_url` in it.
+
+    Returns the decoded JSON, or None if a browser cannot run here, the
+    challenge does not clear, or the API still refuses. Never raises: the
+    caller's contract is that a supplementary failure cannot fail the record.
+    """
+    landing_template = host_uses_waf(api_url)
+    if not landing_template:
+        return None
+
+    from .supplement_atypon import (_start_xvfb, _stop_xvfb,
+                                    headed_browser_available)
+
+    if not headed_browser_available():
+        ctx.log("    repository WAF: no display and no xvfb; cannot solve challenge")
+        return None
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError:
+        ctx.log("    repository WAF: playwright not installed; cannot solve challenge")
+        return None
+
+    virtual_display = None
+    try:
+        virtual_display = _start_xvfb(ctx)
+        with sync_playwright() as driver:
+            browser = driver.chromium.launch(headless=False)  # load-bearing
+            try:
+                page = browser.new_context(
+                    viewport={"width": 1400, "height": 900}).new_page()
+                page.goto(landing_template.format(doi=doi),
+                          wait_until="domcontentloaded", timeout=60000)
+
+                deadline = time.time() + _CHALLENGE_TIMEOUT_S
+                while time.time() < deadline:
+                    time.sleep(1)
+                    content = _page_content(page)
+                    if content and not any(m in content for m in _CHALLENGE_MARKERS):
+                        break
+                else:
+                    ctx.log("    repository WAF: challenge did not clear")
+                    return None
+
+                response = page.request.get(api_url, timeout=60000)
+                if response.status != 200:
+                    ctx.log(f"    repository WAF: API still {response.status} after challenge")
+                    return None
+                return json.loads(response.text())
+            finally:
+                browser.close()
+    except Exception as exc:                       # noqa: BLE001 - never fail the record
+        ctx.log(f"    repository WAF: browser fallback failed ({type(exc).__name__})")
+        return None
+    finally:
+        _stop_xvfb(virtual_display)