fetchpdf 0.1.0__py3-none-any.whl

fetchpdf/__init__.py

@@ -0,0 +1,49 @@
+"""
+fetchpdf: A comprehensive tool to download PDFs from DOIs using multiple sources.
+
+This package provides functions to download academic papers (PDFs) from DOIs using
+multiple fallback sources including OpenAlex, Unpaywall, PubMed Central, Crossref,
+Europe PMC, Semantic Scholar
+"""
+
+from .fetchpdf import fetch_pdf, batch_fetch_pdfs
+from .fetch_metadata_from_doi import fetch_metadata_from_doi
+
+# Deprecated alias. `fetch_pdf_from_doi` was the old name for both this function
+# and the module it lives in; both are retired in favour of `fetch_pdf` /
+# `fetchpdf.fetchpdf`. Kept so existing callers keep working -- there is no
+# behavioural difference, it is the same object.
+fetch_pdf_from_doi = fetch_pdf
+
+__version__ = "0.1.0"
+__all__ = [
+    "fetch_pdf",
+    "fetch_pdf_from_doi",  # deprecated alias for fetch_pdf
+    "batch_fetch_pdfs",
+    "fetch_metadata_from_doi",
+    "pull_supplementary_for",
+]
+
+
+def pull_supplementary_for(*args, **kwargs):
+    """Fetch every supplementary file for one record, beside its main artifact.
+
+    A separate call rather than a keyword on fetch_pdf_from_doi, deliberately: the
+    tiered engine re-enters that function at T5 with a temporary save_path, so a
+    flag threaded through it would fire on the re-entry and write supplementary
+    siblings next to a file that is about to be deleted. The two-call shape makes
+    that structurally impossible.
+
+        from fetchpdf import fetch_pdf_from_doi, pull_supplementary_for
+
+        path = fetch_pdf_from_doi("10.1371/journal.pone.0000308", "out/paper.pdf")
+        summary = pull_supplementary_for("10.1371/journal.pone.0000308",
+                                         doi="10.1371/journal.pone.0000308",
+                                         save_path="out/paper.pdf")
+
+    Imported lazily so the retrieval package is untouched unless it is used. See
+    fetchpdf.retrieval.supplementary.pull_for_record for the full signature.
+    """
+    from .retrieval.supplementary import pull_for_record
+
+    return pull_for_record(*args, **kwargs)

fetchpdf/__main__.py

@@ -0,0 +1,6 @@
+"""Allow `python -m fetchpdf` as an alias for the `fetchpdf` console script."""
+
+from fetchpdf.fetchpdf import main
+
+if __name__ == "__main__":
+    main()

fetchpdf/_env.py

@@ -0,0 +1,63 @@
+"""Single source of truth for .env.local loading and API credentials.
+
+Imported for its side effects at package import time, not lazily: callers such
+as birds_eye_review_code/5_download_pdfs.py load their own .env before
+importing fetchpdf precisely because these are read at import. Moving them to
+call time would silently degrade those callers to no-email API calls.
+"""
+
+import os
+from pathlib import Path
+
+from dotenv import load_dotenv, find_dotenv
+
+
+def _load_env_file():
+    """Locate and load .env.local, returning its path (or None).
+
+    Repo root is tried first so an editable install keeps resolving exactly as
+    it always has. The CWD search is the fallback that makes a real (non-editable)
+    install work at all: there, __file__ points into site-packages/ and the
+    repo-root path simply does not exist, so every key silently came back None.
+    """
+    repo_root = Path(__file__).resolve().parent.parent / ".env.local"
+    if repo_root.exists():
+        load_dotenv(repo_root)
+        return str(repo_root)
+
+    found = find_dotenv(".env.local", usecwd=True)
+    if found:
+        load_dotenv(found)
+        return found
+
+    return None
+
+
+ENV_FILE = _load_env_file()
+
+EMAIL = os.getenv("EMAIL")
+
+# Rate-limit lifts these keys buy, where the provider documents one:
+S2_API_KEY = os.getenv("SEMANTIC_SCHOLAR_API_KEY")    # Semantic Scholar: 1 -> 100 req/s
+ENTREZ_API_KEY = os.getenv("ENTREZ_EUTILS_API_KEY")   # NCBI E-utils: 3 -> 10 req/s
+OPENALEX_API_KEY = os.getenv("OPENALEXAPIKEY")
+CORE_API_KEY = os.getenv("COREAPIKEY")
+SCOPUS_API_KEY = os.getenv("SCOPUS_API_KEY")
+
+# The retrieval agent's OpenRouter backend. Optional: without it that backend
+# reports itself unavailable and the run falls back to the Claude Code CLI, or
+# to the deterministic rules alone. A missing key is never a failed record.
+OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY")
+
+# Elsevier full-text (TDM) retrieval.
+ELSEVIER_TDM_API_KEY = os.getenv("ELSEVIER_TDM_API_KEY")
+
+
+if not EMAIL:
+    print("\033[93m⚠️  Warning: EMAIL not set in .env.local")
+    print("   Please create .env.local with: EMAIL=your@email.com")
+    print("   This affects:")
+    print("     - Crossref API rate limits (10 req/s with email, 5 req/s without)")
+    print("     - Unpaywall API access (required)")
+    print("     - Europe PMC contact info (optional)")
+    print("   Continuing without email...\033[0m")

fetchpdf/_http.py

@@ -0,0 +1,40 @@
+"""Shared HTTP constants and response checks.
+
+Its own module because fetchpdf imports download_from_aa, so the
+latter cannot import back from the former at module scope without a cycle.
+"""
+
+# Keep the Chrome version current: some hosts (Zenodo among them) 403 stale
+# Chrome UAs as a bot signal, which silently breaks every source on that host.
+#
+# Defined once and referenced everywhere -- previously four copies drifted apart,
+# leaving three Playwright/requests paths on Chrome/120 while the main header
+# said 131, so the bot-signal problem this string exists to avoid still applied
+# to exactly the browser-driven sources most likely to be fingerprinted.
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/131.0.0.0 Safari/537.36"
+)
+
+
+def elsevier_first_page_only(headers) -> bool:
+    """True when Elsevier's X-ELS-Status header marks a first-page-only PDF.
+
+    A key entitled to a record's XML but not its PDF does not get an error
+    from `Accept: application/pdf` -- it gets a genuine typeset FIRST PAGE:
+    HTTP 200, %PDF magic, megabyte-scale, real text layer. Nothing in the body
+    distinguishes it from the full article; the only in-band signal is
+
+        X-ELS-Status: WARNING - Response limited to first page because
+                      requestor not entitled to resource
+
+    (observed 2026-08-16 on PII S0022510X13030578, where the "success" was
+    1 of 8 pages). Matched case-insensitively on key and value: the live API
+    serves the header lowercased over HTTP/2, while requests' own header dict
+    is case-insensitive -- callers pass either.
+    """
+    for key, value in (headers or {}).items():
+        if key.lower() == "x-els-status":
+            return "limited to first page" in str(value).lower()
+    return False