fetchpdf 0.1.0__py3-none-any.whl

fetchpdf/retrieval/sources/publisher_html.py

@@ -0,0 +1,154 @@
+"""T2: publisher and repository HTML full text.
+
+HTML sits second, not last. At most OA publishers the article HTML is generated
+from the same JATS the XML endpoint serves, so it carries real <table> with
+<th>, colspan and rowspan -- fidelity close to T1. Ranking it below PDF, as the
+original ladder did, threw away the second-best structured source in favour of
+the one that has to be reconstructed.
+
+Two routes here:
+
+  * Unpaywall oa_locations[]. The `url` field is not always a PDF -- it is
+    frequently the publisher's HTML full text, and sometimes XML. Inspecting
+    what comes back rather than assuming from the field name is what makes this
+    a T2 source at all. publishedVersion is preferred over acceptedVersion, and
+    publisher hosts over repositories, because repository copies are more often
+    author manuscripts whose tables have been reflowed.
+
+  * Plain DOI resolution to whatever the publisher serves.
+
+A headless-browser retry is available but deliberately narrow: it runs only when
+the static fetch returned a page that has <table> elements with no populated
+cells, which is the signature of tables loaded by JS or hosted in a separate
+table viewer. It reuses the existing --add-playwright flag rather than adding a
+dependency or a new switch.
+"""
+
+from typing import List, Optional
+
+from ..artifact import Artifact
+from ..tiers import Tier
+from ..validate import has_empty_table_containers
+
+UNPAYWALL = "https://api.unpaywall.org/v2/{doi}"
+DOI_RESOLVER = "https://doi.org/{doi}"
+
+_VERSION_RANK = {"publishedVersion": 0, "acceptedVersion": 1, "submittedVersion": 2}
+_HOST_RANK = {"publisher": 0, "repository": 1}
+
+
+def fetch_unpaywall_fulltext(ids, ctx) -> Optional[Artifact]:
+    """Walk oa_locations looking for HTML or XML full text, best version first."""
+    if not ids.doi:
+        return None
+
+    # Shared with the resolver, which reads the same oa_locations[] to harvest an
+    # arXiv id. Whichever runs first pays for the request; the other gets it free.
+    data = ctx.resolver.unpaywall(ids)
+    if not data:
+        return None
+
+    locations = data.get("oa_locations") or []
+    best = data.get("best_oa_location")
+    if best and best not in locations:
+        locations = [best] + list(locations)
+    if not locations:
+        return None
+
+    for location in sorted(locations, key=_location_rank):
+        # url_for_pdf is explicitly a PDF; this source is looking for the
+        # landing/full-text URL, which is where the HTML lives.
+        url = location.get("url_for_landing_page") or location.get("url")
+        if not url or url == location.get("url_for_pdf"):
+            continue
+        artifact = _fetch_and_wrap(
+            ctx, url, ids, source="unpaywall_html",
+            license_=location.get("license"),
+            extra={
+                "version": location.get("version"),
+                "host_type": location.get("host_type"),
+            },
+        )
+        if artifact is not None:
+            return artifact
+    return None
+
+
+def fetch_publisher_html(ids, ctx) -> Optional[Artifact]:
+    """Whatever the DOI resolves to, if it turns out to be full text with tables."""
+    if not ids.doi:
+        return None
+    return _fetch_and_wrap(
+        ctx, DOI_RESOLVER.format(doi=ids.doi), ids,
+        source="publisher_html", license_=ids.license,
+    )
+
+
+def _location_rank(location: dict):
+    return (
+        _VERSION_RANK.get(location.get("version"), 3),
+        _HOST_RANK.get(location.get("host_type"), 2),
+    )
+
+
+def _fetch_and_wrap(ctx, url, ids, source, license_=None, extra=None) -> Optional[Artifact]:
+    response = ctx.http.get(url, timeout=45, polite=False)
+    if not response.ok or not response.content:
+        return None
+
+    content = response.content
+    notes: List[str] = []
+
+    if has_empty_table_containers(content):
+        rendered = _render_with_playwright(ctx, url)
+        if rendered:
+            content = rendered
+            notes.append("re-fetched with headless browser: tables were JS-loaded")
+        else:
+            notes.append("tables present but unpopulated; no browser retry available")
+
+    return Artifact(
+        content=content,
+        tier=Tier.T2_HTML,
+        source=source,
+        url=response.url,
+        http_status=response.status,
+        served_content_type=response.content_type,
+        identifier_used=ids.doi,
+        license=license_ or ids.license,
+        extra=dict(extra or {}, notes=notes),
+    )
+
+
+def _render_with_playwright(ctx, url: str) -> Optional[bytes]:
+    """Render the page in the browser this package already ships with.
+
+    Gated on --add-playwright, and only reached when a static fetch produced
+    table containers with no cells. Anything broader would put a browser launch
+    in the hot path of every record.
+    """
+    if not ctx.use_playwright:
+        return None
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError:
+        return None
+
+    from ..._http import USER_AGENT
+
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(
+                headless=True, args=["--no-sandbox", "--disable-setuid-sandbox"]
+            )
+            try:
+                page = browser.new_context(user_agent=USER_AGENT).new_page()
+                page.set_default_timeout(30000)
+                page.goto(url, wait_until="networkidle", timeout=45000)
+                html = page.content()
+            finally:
+                browser.close()
+        return html.encode("utf-8") if html else None
+    except Exception as e:
+        ctx.log(f"    browser render failed: {str(e)[:100]}")
+        return None

fetchpdf/retrieval/sources/supplements.py

@@ -0,0 +1,183 @@
+"""T4: structured supplementary data.
+
+Frequently the actual underlying data rather than a rendering of it -- a
+spreadsheet of per-arm outcomes beats any reconstruction of the typeset table.
+Coverage is partial and mapping supplements back to the paper's tables is
+manual, which is why this sits below LaTeX rather than above it.
+
+The repository routes here are nearly free: the existing Figshare and Zenodo
+handlers already fetch the complete file listing and then discard everything
+that is not a PDF. Filtering that same listing on mimetype and extension costs
+one predicate.
+
+Europe PMC's supplementaryFiles endpoint is the best of the four because it is
+keyed on the PMCID phase 1 already resolved: GET /{PMCID}/supplementaryFiles
+returns 200 application/zip (verified, 132 KB for PMC1817752).
+
+The listings themselves now live in ../supplement_index.py, shared with
+--pull-supplementary. The two callers want different subsets and the difference
+is exactly one predicate, applied here and nowhere else: _is_structured. This
+tier takes only what survives to an extraction pass with its rows and columns
+intact, because here a supplement stands *in place of* the paper's full text --
+a supplementary PDF is no better than the PDF it replaced. --pull-supplementary
+takes everything, because there a supplement stands alongside the paper.
+"""
+
+import io
+import zipfile
+from typing import List, Optional, Tuple
+
+from ..artifact import Artifact
+from ..supplement_index import (
+    EPMC_SUPPL,
+    enumerate_figshare,
+    enumerate_osf,
+    enumerate_zenodo,
+    match_id,
+)
+from ..tiers import Tier
+
+#: Extensions whose row/column structure survives to the extraction pass.
+STRUCTURED_EXTENSIONS = (".xlsx", ".xls", ".csv", ".tsv", ".docx", ".ods", ".txt")
+STRUCTURED_MIMETYPES = (
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.ms-excel",
+    "text/csv",
+    "text/tab-separated-values",
+)
+
+_MAX_FILES = 25
+
+
+def fetch_epmc_supplements(ids, ctx) -> Optional[Artifact]:
+    """The EPMC bundle, stored verbatim as one .suppl.zip.
+
+    Deliberately still its own request rather than going through
+    supplement_index: this tier stores the archive exactly as served, so it wants
+    the bytes, not a listing, and it wants them under the endpoint's default
+    parameters. --pull-supplementary asks the same endpoint with
+    includeInlineImage=no, which returns a different (better, for its purpose)
+    archive -- 4 members instead of 14 on PMC1817752. Sharing one call between
+    the two would mean silently changing what this tier writes.
+    """
+    if not ids.pmcid:
+        return None
+    url = EPMC_SUPPL.format(pmcid=ids.pmcid)
+    response = ctx.http.get(url, timeout=60)
+    if not response.ok or not response.content:
+        ctx.log(f"    EPMC supplements: HTTP {response.status}")
+        return None
+    if not _looks_structured_zip(response.content):
+        ctx.log("    EPMC supplements: bundle holds nothing structured")
+        return None
+    return Artifact(
+        content=response.content,
+        tier=Tier.T4_SUPPLEMENT,
+        source="europepmc_supplements",
+        url=response.url,
+        http_status=response.status,
+        served_content_type=response.content_type,
+        identifier_used=ids.pmcid,
+        license=ids.license,
+        extension=".suppl.zip",
+    )
+
+
+def fetch_figshare_files(ids, ctx) -> Optional[Artifact]:
+    article_id = match_id(ids.doi, r"figshare\.(\d+)")
+    if not article_id:
+        return None
+    files = enumerate_figshare(ids, ctx, with_item_type=False)
+    return _bundle(ctx, _structured_only(files), ids, "figshare_files", article_id)
+
+
+def fetch_zenodo_files(ids, ctx) -> Optional[Artifact]:
+    record_id = match_id(ids.doi, r"zenodo\.(\d+)")
+    if not record_id:
+        return None
+    return _bundle(ctx, _structured_only(enumerate_zenodo(ids, ctx)), ids,
+                   "zenodo_files", record_id)
+
+
+def fetch_osf_files(ids, ctx) -> Optional[Artifact]:
+    osf_id = match_id(ids.doi, r"osf\.io/([a-z0-9]+)") or match_id(ids.doi, r"/([a-z0-9]{5})$")
+    if not osf_id:
+        return None
+    return _bundle(ctx, _structured_only(enumerate_osf(ids, ctx)), ids,
+                   "osf_files", osf_id)
+
+
+# -- shared -----------------------------------------------------------------
+
+
+def _structured_only(files) -> List[Tuple[str, str]]:
+    """The one predicate that separates this tier from --pull-supplementary."""
+    return [(f.name, f.url) for f in files if _is_structured(f.name, f.mimetype)]
+
+
+def _is_structured(name: Optional[str], mimetype: Optional[str]) -> bool:
+    """Filter on mimetype/extension rather than assuming PDF."""
+    if mimetype and str(mimetype).lower() in STRUCTURED_MIMETYPES:
+        return True
+    lowered = (name or "").lower()
+    return any(lowered.endswith(ext) for ext in STRUCTURED_EXTENSIONS)
+
+
+def _bundle(ctx, candidates, ids, source, identifier) -> Optional[Artifact]:
+    """Download the structured files and return them as one artifact.
+
+    A single file keeps its own extension so the downstream reader does not have
+    to unwrap a one-entry zip; several are bundled so a record's supplements
+    stay together and the artifact stays a single addressable thing.
+    """
+    candidates = [(n, u) for n, u in candidates if u][:_MAX_FILES]
+    if not candidates:
+        return None
+
+    downloaded = []
+    for name, url in candidates:
+        response = ctx.http.get(url, timeout=60, polite=False)
+        if response.ok and response.content:
+            downloaded.append((name or "supplement", response.content))
+    if not downloaded:
+        return None
+
+    if len(downloaded) == 1:
+        name, content = downloaded[0]
+        extension = "." + name.rsplit(".", 1)[-1].lower() if "." in name else ".suppl"
+        return Artifact(
+            content=content,
+            tier=Tier.T4_SUPPLEMENT,
+            source=source,
+            url=candidates[0][1],
+            http_status=200,
+            identifier_used=str(identifier),
+            license=ids.license,
+            extension=extension,
+            extra={"files": [name]},
+        )
+
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
+        for name, content in downloaded:
+            archive.writestr(name, content)
+    return Artifact(
+        content=buffer.getvalue(),
+        tier=Tier.T4_SUPPLEMENT,
+        source=source,
+        url=candidates[0][1],
+        http_status=200,
+        identifier_used=str(identifier),
+        license=ids.license,
+        extension=".suppl.zip",
+        extra={"files": [n for n, _ in downloaded]},
+    )
+
+
+def _looks_structured_zip(content: bytes) -> bool:
+    try:
+        with zipfile.ZipFile(io.BytesIO(content)) as archive:
+            return any(_is_structured(name, None) for name in archive.namelist())
+    except (zipfile.BadZipFile, OSError):
+        return False

fetchpdf/retrieval/supplement_atypon.py

@@ -0,0 +1,269 @@
+"""Supplementary files from Atypon-hosted journals, via a real browser tab.
+
+Europe PMC holds supplements for many non-open-access articles and is not
+permitted to serve them: `supplementaryFiles` answers HTTP 200 wrapping
+`<errorBean>...Article with id PMC... is not open access one</errorBean>`.
+Those files still exist on the publisher's site, and for PNAS and its Atypon
+siblings they are reachable -- but only from something that behaves like a
+person with a browser open.
+
+What was measured against 10.1073/pnas.1209746109, all on the same machine and
+network:
+
+    direct GET of the SI url (curl, guessed name)         403
+    Playwright headless, GET the real url                 403
+    Playwright headless, in-page fetch() with 6 cookies   403
+    Playwright headless, persistent profile, click        download canceled
+    Playwright HEADED (xvfb), click the link              261 KB %PDF   <-- works
+    same, fresh profile, on an open-access control        341 KB %PDF   <-- works
+
+Two things are load-bearing and neither is obvious:
+
+  * **Headed, not headless.** This is the discriminating factor. Cookies, a
+    persistent profile, a referer and a browser user-agent were all tried
+    headless and all refused. On a headless server that means xvfb.
+  * **Click, do not navigate.** A direct navigation to the same URL still 403s
+    even when headed. The download has to originate from a user gesture on the
+    listing page.
+
+The redirect chain explains why: `302 -> ?cookieSet=1 -> 302 ->
+/action/cookieAbsent` is Atypon's cookie handshake behind Cloudflare, and it
+only completes for a client that looks entirely ordinary.
+
+This is expensive -- one browser per record -- so it is deliberately the last
+resort, and the caller only reaches it for the handful of records EPMC has
+already said it is withholding.
+"""
+
+import os
+import re
+import shutil
+import tempfile
+from typing import List, Optional
+
+from .supplement_index import ROLE_SUPPLEMENT, SupplementFile
+
+#: Publishers on the Atypon platform, whose SI lives at /doi/suppl/<doi> and
+#: whose files hang off a[href*="suppl_file"]. PNAS is the case that prompted
+#: this; the others share the platform and the same URL shape.
+_ATYPON_HOSTS = {
+    "10.1073": "https://www.pnas.org",          # PNAS
+    "10.1126": "https://www.science.org",       # Science / AAAS
+    "10.1146": "https://www.annualreviews.org",  # Annual Reviews
+    "10.1177": "https://journals.sagepub.com",  # SAGE
+    "10.1080": "https://www.tandfonline.com",   # Taylor & Francis
+}
+
+_SUPPL_PATH = "{host}/doi/suppl/{doi}"
+
+#: Long enough for Cloudflare's handshake plus a real page render. The listing
+#: page took ~5 s to settle in testing; the download itself is fast.
+_PAGE_TIMEOUT_MS = 90000
+_SETTLE_MS = 5000
+_DOWNLOAD_TIMEOUT_MS = 45000
+
+#: One browser per record is the whole cost of this provider, so cap how many
+#: files a single record may pull through it.
+_MAX_FILES = 12
+
+
+def atypon_host_for(doi: str) -> Optional[str]:
+    """The publisher base URL for a DOI, or None if not an Atypon journal."""
+    if not doi:
+        return None
+    prefix = str(doi).split("/", 1)[0].strip()
+    return _ATYPON_HOSTS.get(prefix)
+
+
+def headed_browser_available() -> bool:
+    """Whether a headed Chromium can actually be launched here.
+
+    On Linux that needs a display: either a real one ($DISPLAY) or xvfb. On
+    Windows and macOS headed mode works directly. Returning False makes the
+    caller degrade to its warning rather than raising.
+    """
+    if os.name == "nt" or os.sys.platform == "darwin":
+        return True
+    if os.environ.get("DISPLAY"):
+        return True
+    return shutil.which("xvfb-run") is not None or shutil.which("Xvfb") is not None
+
+
+def _start_xvfb(ctx):
+    """A virtual X display for headed Chromium, or None.
+
+    Returns the Popen so the caller can stop it. Uses a high display number to
+    avoid colliding with a real session, and verifies the server actually came
+    up rather than assuming.
+    """
+    import subprocess
+    import time as _time
+    xvfb = shutil.which("Xvfb")
+    if not xvfb:
+        return None
+    for display in range(99, 110):
+        lock = f"/tmp/.X{display}-lock"
+        if os.path.exists(lock):
+            continue
+        try:
+            proc = subprocess.Popen(
+                [xvfb, f":{display}", "-screen", "0", "1400x900x24", "-nolisten", "tcp"],
+                stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        except OSError:
+            return None
+        _time.sleep(1.0)
+        if proc.poll() is None:
+            os.environ["DISPLAY"] = f":{display}"
+            return proc
+        return None
+    return None
+
+
+def _stop_xvfb(proc) -> None:
+    if proc is None:
+        return
+    try:
+        proc.terminate()
+        proc.wait(timeout=5)
+    except Exception:
+        try:
+            proc.kill()
+        except Exception:
+            pass
+    os.environ.pop("DISPLAY", None)
+
+
+def fetch_atypon_supplements(ids, ctx) -> List[SupplementFile]:
+    """Download SI from an Atypon publisher into a temp dir, as local files.
+
+    Unlike every other enumerator this cannot return remote URLs for the
+    engine to fetch: the URLs only work from inside the browser session that
+    clicked them. So the files are downloaded here and handed back as
+    `file://` entries the normal pipeline can take.
+    """
+    doi = getattr(ids, "doi", None)
+    host = atypon_host_for(doi)
+    if not host:
+        return []
+    if not headed_browser_available():
+        ctx.log("    atypon SI: no display and no xvfb; skipping headed fetch")
+        return []
+
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError:
+        ctx.log("    atypon SI: playwright not installed")
+        return []
+
+    # Headed Chromium needs a display. On a headless Linux box there is none,
+    # so start a virtual one for the duration -- Playwright reads $DISPLAY at
+    # launch. Windows/macOS need nothing.
+    virtual_display = None
+    if os.name != "nt" and os.sys.platform != "darwin" \
+            and not os.environ.get("DISPLAY"):
+        virtual_display = _start_xvfb(ctx)
+        if virtual_display is None:
+            ctx.log("    atypon SI: could not start a virtual display")
+            return []
+
+    listing = _SUPPL_PATH.format(host=host, doi=doi)
+    scratch = tempfile.mkdtemp(prefix=".fetchpdf-atypon-")
+    files: List[SupplementFile] = []
+
+    try:
+        with sync_playwright() as p:
+            context = p.chromium.launch_persistent_context(
+                os.path.join(scratch, "profile"),
+                headless=False,          # load-bearing; see module docstring
+                accept_downloads=True,
+                args=["--no-sandbox", "--disable-setuid-sandbox",
+                      "--disable-blink-features=AutomationControlled"],
+                viewport={"width": 1400, "height": 900},
+                user_agent=("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+                            "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"),
+            )
+            try:
+                page = context.pages[0] if context.pages else context.new_page()
+                page.add_init_script(
+                    "Object.defineProperty(navigator,'webdriver',{get:()=>undefined});")
+                response = page.goto(listing, wait_until="domcontentloaded",
+                                     timeout=_PAGE_TIMEOUT_MS)
+                status = response.status if response else None
+                if status != 200:
+                    ctx.log(f"    atypon SI: listing page HTTP {status}")
+                    return []
+                page.wait_for_timeout(_SETTLE_MS)
+
+                links = page.eval_on_selector_all(
+                    'a[href*="suppl_file"]', "els => els.map(e => e.href)")
+                if not links:
+                    ctx.log("    atypon SI: no suppl_file links on the listing page")
+                    return []
+
+                for index, href in enumerate(dict.fromkeys(links)):
+                    if index >= _MAX_FILES:
+                        ctx.log(f"    atypon SI: stopping at {_MAX_FILES} files")
+                        break
+                    name = href.rsplit("/", 1)[-1].split("?")[0] or f"si_{index+1}"
+                    try:
+                        # A click, not a navigation -- see the module docstring.
+                        # Selected by nth match rather than by full href: the
+                        # DOM href is often relative while eval returns it
+                        # absolute, so an [href="<absolute>"] selector matches
+                        # nothing and times out.
+                        with page.expect_download(
+                                timeout=_DOWNLOAD_TIMEOUT_MS) as download:
+                            page.locator('a[href*="suppl_file"]').nth(index).click(
+                                timeout=20000)
+                        saved = os.path.join(scratch, name)
+                        download.value.save_as(saved)
+                    except Exception as error:
+                        ctx.log(f"    atypon SI: {name}: "
+                                f"{str(error).splitlines()[0][:80]}")
+                        continue
+                    if not os.path.exists(saved) or os.path.getsize(saved) == 0:
+                        continue
+                    files.append(SupplementFile(
+                        name=name,
+                        url="file://" + saved,
+                        provider="atypon_suppl",
+                        listing_index=index,
+                        size_bytes=os.path.getsize(saved),
+                        role=ROLE_SUPPLEMENT,
+                        origin_doi=doi,
+                        extra={"listing_url": listing, "local": saved,
+                               "scratch_dir": scratch},
+                    ))
+                if files:
+                    ctx.log(f"    atypon SI: {len(files)} file(s) via headed browser")
+            finally:
+                try:
+                    context.close()
+                except Exception:
+                    pass
+    except Exception as error:
+        ctx.log(f"    atypon SI: {str(error).splitlines()[0][:100]}")
+    finally:
+        _stop_xvfb(virtual_display)
+
+    if not files:
+        # Nothing to keep the scratch dir for.
+        shutil.rmtree(scratch, ignore_errors=True)
+    else:
+        # The browser profile is large and is dead weight once the files are
+        # downloaded; the files themselves must survive until the pipeline has
+        # copied them out, so only the profile goes now.
+        shutil.rmtree(os.path.join(scratch, "profile"), ignore_errors=True)
+        ctx.scratch.setdefault("_atypon_scratch_dirs", []).append(scratch)
+    return files
+
+
+def cleanup_scratch(ctx) -> None:
+    """Remove the temp dirs this provider downloaded into.
+
+    Called once the pipeline has copied the files to their real homes. Kept
+    separate from fetch because the `file://` entries must stay readable until
+    then -- deleting on the way out would race the download.
+    """
+    for path in ctx.scratch.pop("_atypon_scratch_dirs", []) or []:
+        shutil.rmtree(path, ignore_errors=True)