pmcfetch 0.1.0__py3-none-any.whl

pmcfetch/ids.py

@@ -0,0 +1,41 @@
+"""PMC identifier normalisation.
+
+Corpus files key papers inconsistently: bare digits (``"10007932"``), lowercase
+(``"pmc10007932"``), or canonical (``"PMC10007932"``). S3 object prefixes use the
+canonical form, so everything is normalised on the way in.
+"""
+
+from __future__ import annotations
+
+import re
+
+from pmcfetch.errors import InvalidPMCIDError
+
+# PMC ids are sequential integers with no minimum width, so match any digit run.
+# Leading zeros are stripped so "PMC0010007932" and "PMC10007932" collapse.
+_PMCID_RE = re.compile(r"^(?:pmc)?0*(\d+)$", re.IGNORECASE)
+
+
+def normalize_pmcid(raw: object) -> str:
+    """Return the canonical ``PMC<digits>`` form.
+
+    Raises :class:`InvalidPMCIDError` for anything that is not a PMC id, so a
+    malformed corpus key fails loudly instead of producing a 404 at fetch time.
+    """
+    s = str(raw or "").strip()
+    if not s:
+        raise InvalidPMCIDError("empty PMC id")
+    m = _PMCID_RE.match(s)
+    if not m:
+        raise InvalidPMCIDError(f"not a PMC id: {s!r}")
+    return f"PMC{m.group(1)}"
+
+
+def try_normalize_pmcid(raw: object) -> str | None:
+    """Non-raising variant; returns ``None`` when *raw* is not a PMC id."""
+    try:
+        return normalize_pmcid(raw)
+    except InvalidPMCIDError:
+        return None
+
+__all__ = ["normalize_pmcid", "try_normalize_pmcid"]

pmcfetch/manifest.py

@@ -0,0 +1,160 @@
+"""Read corpus JSON files into a uniform record stream.
+
+Three JSON shapes are supported, matching what corpus tooling commonly emits:
+
+* assignment map -- ``{"<id>": {"metadata": {"namespace": "cardiology"}}}``
+* full-text dump  -- ``{"<id>": "<full text>"}``
+* extraction out  -- ``{"pmc10007932": {"paper_id": ..., "sections": [...]}}``
+
+A plain JSON array of ids and a newline-delimited ``.txt`` id list also work.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Iterable, Iterator
+
+from pmcfetch.ids import try_normalize_pmcid
+
+DEFAULT_NAMESPACE = "general"
+
+
+@dataclass(frozen=True)
+class CorpusRecord:
+    """One downloadable article."""
+
+    pmcid: str
+    namespace: str = DEFAULT_NAMESPACE
+    source_key: str = ""
+
+
+@dataclass
+class LoadResult:
+    """Records plus the keys that could not be parsed as PMC ids."""
+
+    records: list[CorpusRecord] = field(default_factory=list)
+    skipped: list[str] = field(default_factory=list)
+
+    def __len__(self) -> int:
+        return len(self.records)
+
+    @property
+    def namespaces(self) -> dict[str, list[str]]:
+        """``{namespace: [pmcid, ...]}`` preserving first-seen order."""
+        out: dict[str, list[str]] = {}
+        for rec in self.records:
+            out.setdefault(rec.namespace, []).append(rec.pmcid)
+        return out
+
+    def for_namespace(self, namespace: str | None) -> list[CorpusRecord]:
+        """Records in *namespace*; all records when *namespace* is falsy."""
+        if not namespace:
+            return list(self.records)
+        want = str(namespace).strip()
+        return [r for r in self.records if r.namespace == want]
+
+
+def _namespace_of(value: Any, default: str) -> str:
+    """Pull ``metadata.namespace`` (or a top-level ``namespace``) from a record."""
+    if not isinstance(value, dict):
+        return default
+    meta = value.get("metadata")
+    if isinstance(meta, dict):
+        ns = str(meta.get("namespace") or "").strip()
+        if ns:
+            return ns
+    ns = str(value.get("namespace") or "").strip()
+    return ns or default
+
+
+def iter_records(data: Any, *, default_namespace: str = DEFAULT_NAMESPACE) -> Iterator[CorpusRecord]:
+    """Yield :class:`CorpusRecord` for every recognisable PMC id in *data*."""
+    if isinstance(data, dict):
+        items: Iterable[tuple[Any, Any]] = data.items()
+    elif isinstance(data, list):
+        items = ((x, None) for x in data)
+    else:
+        return
+
+    for key, value in items:
+        pmcid = try_normalize_pmcid(key)
+        if pmcid is None:
+            continue
+        yield CorpusRecord(
+            pmcid=pmcid,
+            namespace=_namespace_of(value, default_namespace),
+            source_key=str(key),
+        )
+
+
+def load_records(
+    path: str | Path,
+    *,
+    default_namespace: str = DEFAULT_NAMESPACE,
+) -> LoadResult:
+    """Load a corpus file into a :class:`LoadResult`.
+
+    ``.txt`` inputs are treated as newline-delimited id lists; anything else is
+    parsed as JSON. Unparseable keys are collected in ``skipped`` rather than
+    raising, so one bad key does not abort a 20k-paper batch.
+    """
+    p = Path(path).expanduser()
+    if p.suffix.lower() == ".txt":
+        return _load_id_list(p, default_namespace=default_namespace)
+
+    data = json.loads(p.read_text(encoding="utf-8"))
+    result = LoadResult()
+    seen: set[str] = set()
+
+    keys: Iterable[Any]
+    if isinstance(data, dict):
+        keys = data.keys()
+    elif isinstance(data, list):
+        keys = data
+    else:
+        keys = []
+
+    for rec in iter_records(data, default_namespace=default_namespace):
+        if rec.pmcid in seen:
+            continue
+        seen.add(rec.pmcid)
+        result.records.append(rec)
+
+    resolved = {r.source_key for r in result.records}
+    for key in keys:
+        sk = str(key)
+        if sk not in resolved and try_normalize_pmcid(sk) is None:
+            result.skipped.append(sk)
+
+    return result
+
+
+def _load_id_list(path: Path, *, default_namespace: str) -> LoadResult:
+    result = LoadResult()
+    seen: set[str] = set()
+    for line in path.read_text(encoding="utf-8").splitlines():
+        s = line.strip()
+        if not s or s.startswith("#") or s.startswith("---"):
+            continue
+        # tolerate "PMC123 | Reason: ..." lines, so a failure log works as input
+        token = s.split("|", 1)[0].strip()
+        pmcid = try_normalize_pmcid(token)
+        if pmcid is None:
+            result.skipped.append(s)
+            continue
+        if pmcid in seen:
+            continue
+        seen.add(pmcid)
+        result.records.append(CorpusRecord(pmcid=pmcid, namespace=default_namespace, source_key=s))
+    return result
+
+
+__all__ = [
+    "DEFAULT_NAMESPACE",
+    "CorpusRecord",
+    "LoadResult",
+    "iter_records",
+    "load_records",
+]

pmcfetch/pdf.py

@@ -0,0 +1,44 @@
+"""PDF validation and the source contract that batch downloads depend on."""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+PDF_MAGIC = b"%PDF"
+
+#: Error pages and truncated bodies are small; a real article is not. Used as
+#: the floor for both download validation and the resume check, so a truncated
+#: leftover is never mistaken for a complete file, and every file written passes
+#: the resume check on the next run.
+DEFAULT_MIN_PDF_BYTES = 1000
+
+#: Default per-request timeout, seconds.
+DEFAULT_TIMEOUT = 60.0
+
+
+def is_pdf(data: bytes, *, min_bytes: int = 0) -> bool:
+    """True when *data* looks like a PDF payload.
+
+    Checks the magic anywhere in the first 1 KiB rather than only at offset 0,
+    because some responses carry a short byte-order mark or stray whitespace
+    before ``%PDF``.
+    """
+    if not data:
+        return False
+    if min_bytes and len(data) < min_bytes:
+        return False
+    return data[:4] == PDF_MAGIC or PDF_MAGIC in data[:1024]
+
+
+@runtime_checkable
+class PdfSource(Protocol):
+    """Anything that can produce PDF bytes for a PMC id.
+
+    :func:`pmcfetch.batch.download_one` needs nothing else, so alternative
+    sources can be dropped in without touching the batch runner.
+    """
+
+    def fetch_pdf(self, pmcid: str) -> bytes: ...
+
+
+__all__ = ["PDF_MAGIC", "DEFAULT_MIN_PDF_BYTES", "DEFAULT_TIMEOUT", "is_pdf", "PdfSource"]

pmcfetch/ratelimit.py

@@ -0,0 +1,53 @@
+"""Process-wide request pacing.
+
+Sleeping inside each worker thread (``time.sleep(random.uniform(0.1, 0.5))``)
+bounds nothing globally: with 10 workers that is still ~30 requests/second in
+the worst case. This limiter serialises the *spacing* across all threads so
+``--workers`` and politeness are independent knobs.
+"""
+
+from __future__ import annotations
+
+import random
+import threading
+import time
+
+
+class MinIntervalLimiter:
+    """Allow at most one acquisition per ``min_interval`` seconds, across threads."""
+
+    def __init__(self, min_interval: float = 0.0, *, jitter: float = 0.0) -> None:
+        if min_interval < 0:
+            raise ValueError("min_interval must be >= 0")
+        if jitter < 0:
+            raise ValueError("jitter must be >= 0")
+        self.min_interval = float(min_interval)
+        self.jitter = float(jitter)
+        self._lock = threading.Lock()
+        self._next_at = 0.0
+
+    def acquire(self, *, sleep=time.sleep, now=time.monotonic) -> float:
+        """Block until the next slot is due. Returns seconds actually slept."""
+        if self.min_interval <= 0 and self.jitter <= 0:
+            return 0.0
+        with self._lock:
+            t = now()
+            wait = max(0.0, self._next_at - t)
+            spacing = self.min_interval + (random.uniform(0.0, self.jitter) if self.jitter else 0.0)
+            self._next_at = max(t, self._next_at) + spacing
+        if wait > 0:
+            sleep(wait)
+        return wait
+
+
+class NullLimiter:
+    """No pacing. Useful in tests and for single-shot fetches."""
+
+    min_interval = 0.0
+    jitter = 0.0
+
+    def acquire(self, *, sleep=time.sleep, now=time.monotonic) -> float:  # noqa: ARG002
+        return 0.0
+
+
+__all__ = ["MinIntervalLimiter", "NullLimiter"]

pmcfetch/s3source.py

@@ -0,0 +1,251 @@
+"""PMC Open Access Subset on AWS S3 — the sanctioned bulk source.
+
+NLM publishes every PMC OA article version in the public, anonymous bucket
+``s3://pmc-oa-opendata``. Since the February 2026 restructure the layout is one
+prefix per article version::
+
+    PMC10007932.1/
+        PMC10007932.1.pdf     <- article PDF (when the licence permits)
+        PMC10007932.1.xml     <- JATS XML
+        PMC10007932.1.txt     <- plain text
+        PMC10007932.1.json    <- metadata
+        <publisher files>     <- figures, supplementary PDFs
+
+No rate limiting, no login, and it is the access route PMC sanctions for
+systematic retrieval.
+
+Needs no third-party packages: S3 is plain anonymous HTTPS, so this module
+uses :mod:`urllib` only.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+import re
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from email.message import Message
+from typing import Callable
+
+from pmcfetch import __version__
+from pmcfetch.pdf import is_pdf
+from pmcfetch.errors import DownloadError, HTTPStatusError, NotAPDFError, TransportError
+from pmcfetch.ids import normalize_pmcid
+from pmcfetch.ratelimit import MinIntervalLimiter, NullLimiter
+
+OA_BUCKET = "pmc-oa-opendata"
+OA_BASE_URL = f"https://{OA_BUCKET}.s3.amazonaws.com"
+_S3_NS = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"}
+_USER_AGENT = f"pmcfetch/{__version__} (+https://pypi.org/project/pmcfetch/)"
+
+#: Matches the canonical per-version object name, e.g. ``PMC10007932.1.pdf``.
+#: Deliberately anchored so publisher supplementary PDFs sitting in the same
+#: prefix (``jgo-14-01-302-coif.pdf``) are never mistaken for the article.
+_ARTICLE_OBJECT_RE = re.compile(r"^PMC(\d+)\.(\d+)/PMC\1\.\2\.(pdf|xml|txt|json)$")
+
+
+class ArticleNotInOASubsetError(DownloadError):
+    """No object prefix for this PMCID — not in the Open Access Subset."""
+
+    retryable = False
+
+
+class PDFNotAvailableError(DownloadError):
+    """Article exists but carries no PDF (licence does not permit redistribution)."""
+
+    retryable = False
+
+
+@dataclass(frozen=True)
+class ArticleVersion:
+    """One resolved article version and the objects available for it."""
+
+    pmcid: str
+    version: int
+    keys: dict[str, str]  # extension -> full S3 key
+
+    @property
+    def prefix(self) -> str:
+        return f"{self.pmcid}.{self.version}"
+
+    def url(self, ext: str) -> str | None:
+        key = self.keys.get(ext)
+        return f"{OA_BASE_URL}/{key}" if key else None
+
+    @property
+    def has_pdf(self) -> bool:
+        return "pdf" in self.keys
+
+
+def _retry_after_seconds(headers: Message | None) -> float | None:
+    """``Retry-After`` in seconds; the rarely used HTTP-date form is ignored."""
+    raw = headers.get("Retry-After") if headers is not None else None
+    try:
+        return max(0.0, float(raw)) if raw else None
+    except ValueError:
+        return None
+
+
+def _http_get(url: str, *, timeout: float) -> bytes:
+    """Anonymous GET returning the body, with typed errors."""
+    req = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT})
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.read()
+    except urllib.error.HTTPError as exc:
+        raise HTTPStatusError(int(exc.code), retry_after=_retry_after_seconds(exc.headers)) from exc
+    except Exception as exc:  # noqa: BLE001 - socket/DNS/TLS
+        raise TransportError(f"{exc.__class__.__name__}: {exc}") from exc
+
+
+def list_article_versions(pmcid: str, *, timeout: float = 60.0, http_get=_http_get) -> list[ArticleVersion]:
+    """List every version of *pmcid* present in the bucket, lowest version first."""
+    canonical = normalize_pmcid(pmcid)
+    by_version: dict[int, dict[str, str]] = {}
+    token = ""
+    while True:
+        # Trailing dot keeps PMC1000 from matching PMC10000000.
+        url = f"{OA_BASE_URL}/?list-type=2&prefix={canonical}."
+        if token:
+            url += f"&continuation-token={urllib.parse.quote(token, safe='')}"
+        raw = http_get(url, timeout=timeout)
+
+        try:
+            root = ET.fromstring(raw)
+        except ET.ParseError as exc:
+            raise TransportError(f"malformed S3 listing for {canonical}: {exc}") from exc
+
+        for node in root.findall("s3:Contents/s3:Key", _S3_NS):
+            key = (node.text or "").strip()
+            m = _ARTICLE_OBJECT_RE.match(key)
+            if not m or f"PMC{m.group(1)}" != canonical:
+                continue
+            by_version.setdefault(int(m.group(2)), {})[m.group(3)] = key
+
+        # S3 pages listings at 1,000 keys. An article with many versions or files
+        # exceeds that, and the newest version can sort onto a later page.
+        if root.findtext("s3:IsTruncated", default="", namespaces=_S3_NS) != "true":
+            break
+        token = root.findtext("s3:NextContinuationToken", default="", namespaces=_S3_NS)
+        if not token:
+            raise TransportError(f"truncated S3 listing for {canonical} has no continuation token")
+
+    return [
+        ArticleVersion(pmcid=canonical, version=v, keys=by_version[v])
+        for v in sorted(by_version)
+    ]
+
+
+def resolve_latest_version(pmcid: str, *, timeout: float = 60.0, http_get=_http_get) -> ArticleVersion:
+    """Highest-numbered version of *pmcid*, or raise :class:`ArticleNotInOASubsetError`."""
+    versions = list_article_versions(pmcid, timeout=timeout, http_get=http_get)
+    if not versions:
+        raise ArticleNotInOASubsetError(
+            f"{normalize_pmcid(pmcid)} is not in the PMC Open Access Subset",
+            pmcid=normalize_pmcid(pmcid),
+        )
+    return versions[-1]
+
+
+class S3OpenAccessClient:
+    """Fetch article PDFs from ``s3://pmc-oa-opendata``.
+
+    Implements the :class:`pmcfetch.pdf.PdfSource` contract
+    (``fetch_pdf(pmcid) -> bytes``), so it drops straight into ``download_one``
+    and ``download_many``.
+    """
+
+    def __init__(
+        self,
+        *,
+        timeout: float = 60.0,
+        retries: int = 3,
+        backoff_base: float = 0.5,
+        backoff_cap: float = 15.0,
+        limiter: MinIntervalLimiter | NullLimiter | None = None,
+        http_get: Callable[..., bytes] = _http_get,
+        sleep: Callable[[float], None] = time.sleep,
+    ) -> None:
+        self.timeout = float(timeout)
+        self.retries = max(0, int(retries))
+        self.backoff_base = float(backoff_base)
+        self.backoff_cap = float(backoff_cap)
+        self.limiter = limiter if limiter is not None else NullLimiter()
+        self._http_get = http_get
+        self._sleep = sleep
+
+    def _get(self, url: str) -> bytes:
+        """GET with retry on 429, 5xx and transport errors."""
+        total = self.retries + 1
+        for attempt in range(total):
+            self.limiter.acquire(sleep=self._sleep)
+            try:
+                return self._http_get(url, timeout=self.timeout)
+            except DownloadError as exc:
+                if not exc.retryable or attempt == total - 1:
+                    raise
+                self._sleep(self._backoff(attempt, getattr(exc, "retry_after", None)))
+        raise TransportError(f"exhausted retries for {url}")  # pragma: no cover
+
+    def _backoff(self, attempt: int, retry_after: float | None) -> float:
+        """Full-jitter exponential delay, stretched to any ``Retry-After``, capped at ``backoff_cap``."""
+        delay = random.uniform(0.0, min(self.backoff_cap, self.backoff_base * (2**attempt)))
+        if retry_after is not None:
+            delay = max(delay, retry_after)
+        return min(delay, self.backoff_cap)
+
+    def metadata(self, pmcid: str) -> dict:
+        """Article metadata JSON (doi, pmid, licence, is_retracted, urls…)."""
+        version = resolve_latest_version(
+            pmcid, timeout=self.timeout, http_get=lambda u, timeout: self._get(u)
+        )
+        url = version.url("json")
+        if not url:
+            raise ArticleNotInOASubsetError(
+                f"{version.prefix} has no metadata object", pmcid=version.pmcid
+            )
+        return json.loads(self._get(url).decode("utf-8"))
+
+    def fetch_pdf(self, pmcid: str) -> bytes:
+        """PDF bytes for the latest version of *pmcid*."""
+        version = resolve_latest_version(
+            pmcid, timeout=self.timeout, http_get=lambda u, timeout: self._get(u)
+        )
+        url = version.url("pdf")
+        if not url:
+            raise PDFNotAvailableError(
+                f"{version.prefix} has no PDF in the OA subset "
+                "(publisher licence does not permit PDF redistribution); "
+                "XML and plain text are still available",
+                pmcid=version.pmcid,
+            )
+        data = self._get(url)
+        if not is_pdf(data):
+            raise NotAPDFError(pmcid=version.pmcid, head=data[:64])
+        return data
+
+    def close(self) -> None:
+        """No persistent connection to release; present for API symmetry."""
+
+    def __enter__(self) -> "S3OpenAccessClient":
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.close()
+
+
+__all__ = [
+    "OA_BUCKET",
+    "OA_BASE_URL",
+    "ArticleVersion",
+    "ArticleNotInOASubsetError",
+    "PDFNotAvailableError",
+    "S3OpenAccessClient",
+    "list_article_versions",
+    "resolve_latest_version",
+]