pmcfetch 0.1.0__py3-none-any.whl

pmcfetch/__init__.py

@@ -0,0 +1,102 @@
+"""Targeted PDF retrieval from the PMC Open Access Subset.
+
+NLM publishes every PMC Open Access article version in the public, anonymous
+bucket ``s3://pmc-oa-opendata``. No AWS account, no credentials, no rate limit,
+and it is the access route PMC sanctions for systematic retrieval.
+
+``aws s3 sync`` is all-or-nothing — there is no way to pull just the articles you
+care about. This package fills that gap: give it a list of PMC ids and it
+resolves each article's version, picks the article PDF (not the publisher's
+supplementary PDFs sitting in the same prefix), and writes the results with
+resume, retries, atomic writes, and per-namespace layout.
+
+Zero third-party dependencies.
+
+    from pmcfetch import S3OpenAccessClient, load_records, download_many
+
+    records = load_records("corpus.json").for_namespace("cardiology")
+    with S3OpenAccessClient() as client:
+        report = download_many(records, "pdfs/", client=client, workers=16)
+    print(report.as_dict())
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from pmcfetch.batch import (
+    BatchReport,
+    DownloadResult,
+    FailureLog,
+    Outcome,
+    download_many,
+    download_one,
+    target_path,
+    write_pdf_atomic,
+)
+from pmcfetch.errors import (
+    DownloadError,
+    HTTPStatusError,
+    InvalidPMCIDError,
+    NotAPDFError,
+    PMCFetchError,
+    TransportError,
+)
+from pmcfetch.ids import normalize_pmcid, try_normalize_pmcid
+from pmcfetch.manifest import CorpusRecord, LoadResult, iter_records, load_records
+from pmcfetch.pdf import DEFAULT_MIN_PDF_BYTES, DEFAULT_TIMEOUT, PdfSource, is_pdf
+from pmcfetch.ratelimit import MinIntervalLimiter, NullLimiter
+from pmcfetch.s3source import (
+    OA_BASE_URL,
+    OA_BUCKET,
+    ArticleNotInOASubsetError,
+    ArticleVersion,
+    PDFNotAvailableError,
+    S3OpenAccessClient,
+    list_article_versions,
+    resolve_latest_version,
+)
+
+__all__ = [
+    "__version__",
+    # source
+    "S3OpenAccessClient",
+    "OA_BUCKET",
+    "OA_BASE_URL",
+    "ArticleVersion",
+    "list_article_versions",
+    "resolve_latest_version",
+    # batch
+    "download_one",
+    "download_many",
+    "BatchReport",
+    "DownloadResult",
+    "Outcome",
+    "FailureLog",
+    "target_path",
+    "write_pdf_atomic",
+    # manifest
+    "load_records",
+    "iter_records",
+    "CorpusRecord",
+    "LoadResult",
+    # ids
+    "normalize_pmcid",
+    "try_normalize_pmcid",
+    # pdf / pacing
+    "PdfSource",
+    "is_pdf",
+    "DEFAULT_MIN_PDF_BYTES",
+    "DEFAULT_TIMEOUT",
+    "MinIntervalLimiter",
+    "NullLimiter",
+    # errors
+    "PMCFetchError",
+    "DownloadError",
+    "HTTPStatusError",
+    "NotAPDFError",
+    "PDFNotAvailableError",
+    "ArticleNotInOASubsetError",
+    "InvalidPMCIDError",
+    "TransportError",
+]

pmcfetch/batch.py

@@ -0,0 +1,278 @@
+"""Concurrent, resumable batch downloads."""
+
+from __future__ import annotations
+
+import os
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+from typing import Callable, Iterable, Sequence
+
+from pmcfetch.pdf import DEFAULT_MIN_PDF_BYTES, PdfSource, is_pdf
+from pmcfetch.errors import NotAPDFError
+from pmcfetch.manifest import CorpusRecord
+
+
+class Outcome(str, Enum):
+    DOWNLOADED = "downloaded"
+    SKIPPED = "skipped"
+    FAILED = "failed"
+
+
+@dataclass
+class DownloadResult:
+    pmcid: str
+    outcome: Outcome
+    path: Path | None = None
+    bytes_written: int = 0
+    reason: str = ""
+
+    @property
+    def ok(self) -> bool:
+        return self.outcome in (Outcome.DOWNLOADED, Outcome.SKIPPED)
+
+
+@dataclass
+class BatchReport:
+    results: list[DownloadResult] = field(default_factory=list)
+
+    @property
+    def downloaded(self) -> int:
+        return sum(1 for r in self.results if r.outcome is Outcome.DOWNLOADED)
+
+    @property
+    def skipped(self) -> int:
+        return sum(1 for r in self.results if r.outcome is Outcome.SKIPPED)
+
+    @property
+    def failed(self) -> int:
+        return sum(1 for r in self.results if r.outcome is Outcome.FAILED)
+
+    @property
+    def total(self) -> int:
+        return len(self.results)
+
+    @property
+    def failures(self) -> list[DownloadResult]:
+        return [r for r in self.results if r.outcome is Outcome.FAILED]
+
+    def as_dict(self) -> dict[str, int]:
+        return {
+            "total": self.total,
+            "downloaded": self.downloaded,
+            "skipped": self.skipped,
+            "failed": self.failed,
+        }
+
+
+class FailureLog:
+    """Thread-safe append-only failure log.
+
+    Each line is ``<pmcid> | Reason: <reason>``, which ``load_records`` reads
+    back as an id list, so a log doubles as the input for a retry run.
+    """
+
+    def __init__(self, path: str | Path | None, *, truncate: bool = False) -> None:
+        self.path = Path(path).expanduser() if path else None
+        self._lock = threading.Lock()
+        if self.path and truncate:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            self.path.write_text("--- Failed Downloads Log ---\n", encoding="utf-8")
+
+    def record(self, pmcid: str, reason: str) -> None:
+        if not self.path:
+            return
+        with self._lock:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            with self.path.open("a", encoding="utf-8") as fh:
+                fh.write(f"{pmcid} | Reason: {reason}\n")
+
+
+def _namespace_dir(namespace: str) -> str:
+    """*namespace* as a single path component, so corpus data cannot escape the output root."""
+    name = namespace.replace("/", "_").replace("\\", "_").replace("\0", "_")
+    return name if name.strip(".") else "_"
+
+
+def target_path(
+    dest_root: str | Path,
+    record: CorpusRecord,
+    *,
+    by_namespace: bool = True,
+) -> Path:
+    """Where *record* should land: ``<root>/<namespace>/PMC<id>.pdf`` by default."""
+    root = Path(dest_root).expanduser()
+    folder = root / _namespace_dir(record.namespace) if by_namespace else root
+    return folder / f"{record.pmcid}.pdf"
+
+
+def _already_complete(path: Path, *, min_bytes: int, verify_magic: bool) -> bool:
+    """True when *path* is a usable PDF already on disk."""
+    try:
+        if not path.is_file() or path.stat().st_size < max(1, min_bytes):
+            return False
+    except OSError:
+        return False
+    if not verify_magic:
+        return True
+    try:
+        with path.open("rb") as fh:
+            return is_pdf(fh.read(1024))
+    except OSError:
+        return False
+
+
+def write_pdf_atomic(path: Path, data: bytes) -> int:
+    """Write *data* via a ``.part`` file + rename.
+
+    Writing straight to the target means an interrupted job (walltime kill,
+    node eviction) leaves a truncated file that a size-based resume check then
+    treats as complete forever. A write that fails removes its ``.part`` file.
+    """
+    path.parent.mkdir(parents=True, exist_ok=True)
+    # Unique per writer, so two threads handed the same record cannot interleave.
+    tmp = path.with_name(f"{path.name}.{os.getpid()}-{threading.get_ident()}.part")
+    try:
+        with tmp.open("wb") as fh:
+            fh.write(data)
+            fh.flush()
+            os.fsync(fh.fileno())
+        tmp.replace(path)
+    except BaseException:
+        tmp.unlink(missing_ok=True)
+        raise
+    return len(data)
+
+
+def _check_body(data: bytes, pmcid: str, min_bytes: int) -> None:
+    """Hold a download to the resume check's standard, so it is skipped next run."""
+    if not is_pdf(data):
+        raise NotAPDFError(pmcid=pmcid, head=data[:64])
+    if len(data) < min_bytes:
+        raise NotAPDFError(
+            f"response body is {len(data)} bytes, below the {min_bytes}-byte minimum",
+            pmcid=pmcid,
+            head=data[:64],
+        )
+
+
+def download_one(
+    client: PdfSource,
+    record: CorpusRecord,
+    dest_root: str | Path,
+    *,
+    by_namespace: bool = True,
+    overwrite: bool = False,
+    verify_existing: bool = True,
+    min_pdf_bytes: int = DEFAULT_MIN_PDF_BYTES,
+    failure_log: FailureLog | None = None,
+) -> DownloadResult:
+    """Fetch one article to disk. Never raises for expected download failures."""
+    path = target_path(dest_root, record, by_namespace=by_namespace)
+
+    if not overwrite and _already_complete(
+        path, min_bytes=min_pdf_bytes, verify_magic=verify_existing
+    ):
+        return DownloadResult(record.pmcid, Outcome.SKIPPED, path=path, reason="already_present")
+
+    def _failed(reason: str) -> DownloadResult:
+        if failure_log:
+            failure_log.record(record.pmcid, reason)
+        return DownloadResult(record.pmcid, Outcome.FAILED, reason=reason)
+
+    try:
+        data = client.fetch_pdf(record.pmcid)
+        _check_body(data, record.pmcid, min_pdf_bytes)
+    except Exception as exc:  # noqa: BLE001 - one bad paper must not kill the batch
+        return _failed(f"{exc.__class__.__name__}: {exc}")
+
+    try:
+        n = write_pdf_atomic(path, data)
+    except OSError as exc:
+        return _failed(f"WriteError: {exc}")
+
+    return DownloadResult(record.pmcid, Outcome.DOWNLOADED, path=path, bytes_written=n)
+
+
+def download_many(
+    records: Sequence[CorpusRecord] | Iterable[CorpusRecord],
+    dest_root: str | Path,
+    *,
+    client: PdfSource,
+    workers: int = 4,
+    by_namespace: bool = True,
+    overwrite: bool = False,
+    verify_existing: bool = True,
+    min_pdf_bytes: int = DEFAULT_MIN_PDF_BYTES,
+    failure_log: FailureLog | None = None,
+    progress: Callable[[DownloadResult, int, int], None] | None = None,
+) -> BatchReport:
+    """Download *records* concurrently.
+
+    Politeness is governed by the client's rate limiter, not by ``workers``;
+    raising ``workers`` increases parallelism of connection setup and disk I/O
+    while the limiter still paces outbound requests.
+
+    *progress* runs on the calling thread as each record finishes. If it raises,
+    or the batch is interrupted (Ctrl-C), queued records are cancelled and only
+    the downloads already in flight are waited for.
+    """
+    items = list(records)
+    report = BatchReport()
+    total = len(items)
+    if total == 0:
+        return report
+
+    n_workers = max(1, int(workers))
+
+    def _run(rec: CorpusRecord) -> DownloadResult:
+        try:
+            return download_one(
+                client,
+                rec,
+                dest_root,
+                by_namespace=by_namespace,
+                overwrite=overwrite,
+                verify_existing=verify_existing,
+                min_pdf_bytes=min_pdf_bytes,
+                failure_log=failure_log,
+            )
+        except Exception as exc:  # noqa: BLE001 - defensive: one bad record must not kill the batch
+            reason = f"{exc.__class__.__name__}: {exc}"
+            if failure_log:
+                failure_log.record(rec.pmcid, reason)
+            return DownloadResult(rec.pmcid, Outcome.FAILED, reason=reason)
+
+    def _collect(i: int, res: DownloadResult) -> None:
+        report.results.append(res)
+        if progress:
+            progress(res, i, total)
+
+    if n_workers == 1:
+        for i, rec in enumerate(items, start=1):
+            _collect(i, _run(rec))
+        return report
+
+    pool = ThreadPoolExecutor(max_workers=n_workers)
+    try:
+        futures = [pool.submit(_run, rec) for rec in items]
+        for i, fut in enumerate(as_completed(futures), start=1):
+            _collect(i, fut.result())
+    finally:
+        # Leaving the pool via ``with`` would run every queued download first.
+        pool.shutdown(wait=True, cancel_futures=True)
+    return report
+
+
+__all__ = [
+    "Outcome",
+    "DownloadResult",
+    "BatchReport",
+    "FailureLog",
+    "target_path",
+    "write_pdf_atomic",
+    "download_one",
+    "download_many",
+]

pmcfetch/cli.py

@@ -0,0 +1,228 @@
+"""``pmcfetch`` command line interface."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from pmcfetch import __version__
+from pmcfetch.batch import DownloadResult, FailureLog, Outcome, download_many
+from pmcfetch.pdf import DEFAULT_MIN_PDF_BYTES, DEFAULT_TIMEOUT
+from pmcfetch.errors import PMCFetchError
+from pmcfetch.manifest import load_records
+from pmcfetch.ratelimit import MinIntervalLimiter, NullLimiter
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="pmcfetch",
+        description="Download open-access PDFs from the PMC Open Access Subset on AWS.",
+    )
+    parser.add_argument("--version", action="version", version=f"pmcfetch {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    dl = sub.add_parser("download", help="Download PDFs listed in a corpus file")
+    dl.add_argument("--corpus", required=True, help="Corpus JSON (or .txt id list)")
+    dl.add_argument("--out", required=True, help="Output root directory")
+    dl.add_argument(
+        "--namespace",
+        default=None,
+        help="Only download this namespace (e.g. cardiology)",
+    )
+    dl.add_argument(
+        "--namespaces",
+        default="",
+        help="Comma-separated namespace order used to resolve --task-id",
+    )
+    dl.add_argument(
+        "--task-id",
+        type=int,
+        default=None,
+        help="1-based array index selecting a namespace (defaults to $SGE_TASK_ID)",
+    )
+    dl.add_argument("--workers", type=int, default=4, help="Concurrent downloads (default 4)")
+    dl.add_argument(
+        "--rps",
+        type=float,
+        default=2.0,
+        help=(
+            "Max outbound requests per second across all workers. The PMC S3 bucket "
+            "is not rate limited, so 0 disables pacing entirely (default 2.0)"
+        ),
+    )
+    dl.add_argument(
+        "--jitter",
+        type=float,
+        default=0.2,
+        help="Extra random spacing per request, seconds (only applies with --rps > 0)",
+    )
+    dl.add_argument("--retries", type=int, default=4, help="Retries per article (default 4)")
+    dl.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT, help="Per-request timeout")
+    dl.add_argument("--limit", type=int, default=0, help="Stop after N articles (0 = all)")
+    dl.add_argument("--min-bytes", type=int, default=DEFAULT_MIN_PDF_BYTES, help="Minimum PDF size")
+    dl.add_argument("--overwrite", action="store_true", help="Re-download files already on disk")
+    dl.add_argument("--flat", action="store_true", help="Write all PDFs into --out, no subfolders")
+    dl.add_argument(
+        "--no-verify-existing",
+        action="store_true",
+        help="Resume on file size alone, skipping the %%PDF magic re-check",
+    )
+    dl.add_argument("--failed-log", default=None, help="Append failures here")
+    dl.add_argument("--json", action="store_true", help="Print the summary as JSON")
+    dl.add_argument("--quiet", action="store_true", help="Suppress per-article progress")
+    dl.add_argument("--dry-run", action="store_true", help="List what would be fetched, then exit")
+
+    ns = sub.add_parser("namespaces", help="Show namespace counts in a corpus file")
+    ns.add_argument("--corpus", required=True)
+    ns.add_argument("--json", action="store_true")
+
+    return parser
+
+
+def _resolve_namespace(args: argparse.Namespace, available: list[str]) -> str | None:
+    """Pick a namespace from --namespace, or from --task-id/$SGE_TASK_ID."""
+    if args.namespace:
+        return str(args.namespace).strip()
+
+    task_id = args.task_id
+    if task_id is None:
+        raw = os.environ.get("SGE_TASK_ID", "").strip()
+        if raw and raw.isdigit():
+            task_id = int(raw)
+    if task_id is None:
+        return None
+
+    ordered = [s.strip() for s in str(args.namespaces or "").split(",") if s.strip()] or available
+    idx = task_id - 1
+    if idx < 0 or idx >= len(ordered):
+        raise PMCFetchError(
+            f"--task-id {task_id} is out of range for {len(ordered)} namespaces: {ordered}"
+        )
+    return ordered[idx]
+
+
+def _build_limiter(rps: float, jitter: float) -> MinIntervalLimiter | NullLimiter:
+    """``--rps 0`` switches pacing off entirely; jitter only widens a positive rate."""
+    if rps <= 0:
+        return NullLimiter()
+    return MinIntervalLimiter(min_interval=1.0 / rps, jitter=max(0.0, jitter))
+
+
+def _cmd_namespaces(args: argparse.Namespace) -> int:
+    loaded = load_records(args.corpus)
+    groups = loaded.namespaces
+    if args.json:
+        print(json.dumps({k: len(v) for k, v in groups.items()}, indent=2))
+        return 0
+    width = max((len(k) for k in groups), default=9)
+    print(f"{'namespace'.ljust(width)}  count")
+    for name, ids in sorted(groups.items(), key=lambda kv: -len(kv[1])):
+        print(f"{name.ljust(width)}  {len(ids)}")
+    print(f"{'TOTAL'.ljust(width)}  {len(loaded.records)}")
+    if loaded.skipped:
+        print(f"\nskipped {len(loaded.skipped)} unparseable key(s), e.g. {loaded.skipped[:3]}")
+    return 0
+
+
+def _cmd_download(args: argparse.Namespace) -> int:
+    loaded = load_records(args.corpus)
+    if loaded.skipped and not args.quiet:
+        print(
+            f"[pmcfetch] skipped {len(loaded.skipped)} unparseable key(s)",
+            file=sys.stderr,
+        )
+
+    namespace = _resolve_namespace(args, sorted(loaded.namespaces))
+    records = loaded.for_namespace(namespace)
+    if args.limit and args.limit > 0:
+        records = records[: args.limit]
+
+    label = namespace or "ALL"
+    if not args.quiet:
+        print(f"[pmcfetch] corpus={args.corpus} namespace={label} n={len(records)}")
+
+    if args.dry_run:
+        for rec in records[:20]:
+            print(f"  would fetch {rec.pmcid}  -> {rec.namespace}/")
+        if len(records) > 20:
+            print(f"  ... and {len(records) - 20} more")
+        return 0
+
+    if not records:
+        print("[pmcfetch] nothing to do", file=sys.stderr)
+        return 0
+
+    limiter = _build_limiter(args.rps, args.jitter)
+    failure_log = FailureLog(args.failed_log) if args.failed_log else None
+
+    def _progress(res: DownloadResult, i: int, total: int) -> None:
+        if args.quiet:
+            return
+        mark = {
+            Outcome.DOWNLOADED: "ok  ",
+            Outcome.SKIPPED: "skip",
+            Outcome.FAILED: "FAIL",
+        }[res.outcome]
+        suffix = f"  {res.reason}" if res.outcome is Outcome.FAILED else ""
+        print(f"[{i}/{total}] {mark} {res.pmcid}{suffix}", flush=True)
+
+    from pmcfetch.s3source import S3OpenAccessClient
+
+    with S3OpenAccessClient(
+        timeout=args.timeout,
+        retries=args.retries,
+        limiter=limiter,
+    ) as client:
+        report = download_many(
+            records,
+            Path(args.out),
+            client=client,
+            workers=args.workers,
+            by_namespace=not args.flat,
+            overwrite=args.overwrite,
+            verify_existing=not args.no_verify_existing,
+            min_pdf_bytes=args.min_bytes,
+            failure_log=failure_log,
+            progress=_progress,
+        )
+
+    if args.json:
+        print(json.dumps({"namespace": label, **report.as_dict()}, indent=2))
+    else:
+        print(
+            f"\n[pmcfetch] {label}: downloaded={report.downloaded} "
+            f"skipped={report.skipped} failed={report.failed} total={report.total}"
+        )
+        if failure_log and report.failed:
+            print(f"[pmcfetch] failures logged to {failure_log.path}")
+
+    return 1 if report.failed else 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+    try:
+        if args.command == "download":
+            return _cmd_download(args)
+        if args.command == "namespaces":
+            return _cmd_namespaces(args)
+    except PMCFetchError as exc:
+        print(f"pmcfetch: {exc}", file=sys.stderr)
+        return 2
+    except json.JSONDecodeError as exc:
+        print(f"pmcfetch: corpus file is not valid JSON: {exc}", file=sys.stderr)
+        return 2
+    except OSError as exc:  # unreadable corpus, unwritable --out or --failed-log
+        print(f"pmcfetch: {exc}", file=sys.stderr)
+        return 2
+    except KeyboardInterrupt:
+        print("\npmcfetch: interrupted", file=sys.stderr)
+        return 130
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pmcfetch/errors.py

@@ -0,0 +1,61 @@
+"""Exception hierarchy for pmcfetch."""
+
+from __future__ import annotations
+
+
+class PMCFetchError(Exception):
+    """Base for every pmcfetch failure."""
+
+
+class InvalidPMCIDError(PMCFetchError, ValueError):
+    """A supplied identifier is not a usable PMC id."""
+
+
+class DownloadError(PMCFetchError):
+    """Base for per-article download failures."""
+
+    retryable: bool = False
+
+    def __init__(self, message: str, *, pmcid: str = "") -> None:
+        self.pmcid = pmcid
+        super().__init__(message)
+
+
+class HTTPStatusError(DownloadError):
+    """Server returned a non-200 status."""
+
+    def __init__(self, status_code: int, *, pmcid: str = "", retry_after: float | None = None) -> None:
+        self.status_code = int(status_code)
+        # Seconds the server asked us to wait (``Retry-After``), when it said.
+        self.retry_after = retry_after
+        # 429 and 5xx are worth another attempt; 403/404 are not.
+        self.retryable = self.status_code == 429 or 500 <= self.status_code < 600
+        super().__init__(f"HTTP {self.status_code}", pmcid=pmcid)
+
+
+class NotAPDFError(DownloadError):
+    """Body came back 200 but is not a usable PDF (error page, truncated object, HTML)."""
+
+    retryable = False
+
+    def __init__(
+        self, message: str = "response body is not a PDF", *, pmcid: str = "", head: bytes = b""
+    ) -> None:
+        self.head = head
+        super().__init__(message, pmcid=pmcid)
+
+
+class TransportError(DownloadError):
+    """Network or TLS level failure; usually worth retrying."""
+
+    retryable = True
+
+
+__all__ = [
+    "PMCFetchError",
+    "InvalidPMCIDError",
+    "DownloadError",
+    "HTTPStatusError",
+    "NotAPDFError",
+    "TransportError",
+]