pmcfetch 0.1.0__tar.gz

pmcfetch-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Kavach Shah
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pmcfetch-0.1.0/MANIFEST.in

@@ -0,0 +1,2 @@
+# setuptools ships tests/test_*.py in the sdist by default, but not the conftest they import.
+recursive-include tests *.py

pmcfetch-0.1.0/PKG-INFO

@@ -0,0 +1,186 @@
+Metadata-Version: 2.4
+Name: pmcfetch
+Version: 0.1.0
+Summary: Targeted PDF retrieval from the PMC Open Access Subset on AWS. Zero dependencies.
+Author: Kavach Shah
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/kms2229/pmcfetch
+Project-URL: Issues, https://github.com/kms2229/pmcfetch/issues
+Keywords: pubmed-central,pmc,open-access,bulk-download,bioinformatics
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Dynamic: license-file
+
+# pmcfetch
+
+Targeted PDF retrieval from the **PMC Open Access Subset**, with resume, retries,
+atomic writes, and namespace-aware batch layout. Zero dependencies.
+
+```bash
+pip install pmcfetch
+pmcfetch download --corpus corpus.json --out pdfs/ --workers 16 --rps 0
+```
+
+Requires Python 3.10+. Nothing outside the standard library.
+
+## Why
+
+NLM publishes every PMC Open Access article version in the public bucket
+`s3://pmc-oa-opendata` — anonymous HTTPS, no AWS account, no credentials, no rate
+limit, and the access route PMC sanctions for systematic retrieval.
+
+But `aws s3 sync` is all-or-nothing. There's no `--from-id-list`, so pulling the
+few thousand articles you care about means pulling all ~8 million. And fetching
+them one by one with `aws s3 cp` requires knowing things that aren't obvious:
+
+- **Which version?** Articles live under `PMC10007932.1/`, `.2/`, sometimes `.10/`.
+  You can't know without listing first — and the sort must be numeric, not
+  lexicographic.
+- **Which PDF?** `PMC10007932.1/` also contains `jgo-14-01-302-coif.pdf`,
+  `-dss.pdf`, `-prf.pdf`. Glob `*.pdf` and you get the wrong file.
+- **Not every article has one.** Some licences permit XML and plain-text
+  redistribution but not the PDF.
+
+`pmcfetch` handles all three, then adds the batch machinery you'd otherwise
+rewrite: resume, atomic writes, concurrency, pacing, and structured failure logs.
+
+## Python API
+
+```python
+from pmcfetch import S3OpenAccessClient, load_records, download_many, FailureLog
+
+records = load_records("corpus.json").for_namespace("cardiology")
+
+with S3OpenAccessClient() as client:
+    report = download_many(
+        records, "pdfs/",
+        client=client,
+        workers=16,
+        failure_log=FailureLog("failed.txt"),
+    )
+
+print(report.as_dict())        # {'total': 3139, 'downloaded': 3104, 'skipped': 0, 'failed': 35}
+for r in report.failures:
+    print(r.pmcid, r.reason)
+```
+
+Single article, plus metadata:
+
+```python
+with S3OpenAccessClient() as c:
+    meta = c.metadata("10007932")
+    # {'pmcid': 'PMC10007932', 'doi': '10.21037/jgo-23-76',
+    #  'license_code': 'CC BY-NC-ND', 'is_retracted': False, ...}
+    if not meta["is_retracted"]:
+        pdf = c.fetch_pdf("10007932")   # accepts 10007932 / pmc10007932 / PMC10007932
+```
+
+`license_code` and `is_retracted` come free with every article — filter on licence
+terms and drop retracted papers before they reach your corpus.
+
+## CLI
+
+```bash
+# what's in a corpus file?
+pmcfetch namespaces --corpus corpus.json
+
+# pull one namespace
+pmcfetch download --corpus corpus.json --out pdfs/ --namespace cardiology --workers 16 --rps 0
+
+# dry run, no requests
+pmcfetch download --corpus corpus.json --out pdfs/ --limit 20 --dry-run
+
+# retry failures — the log doubles as an input list
+pmcfetch download --corpus pdfs/failed.txt --out pdfs/ --flat
+```
+
+Exit code is `1` when any article failed, `0` otherwise — so a batch job fails
+loudly instead of silently producing a short corpus. Bad arguments or an
+unreadable corpus file exit `2`, and Ctrl-C exits `130`.
+
+S3 has no rate limit, so `--rps 0` switches pacing off entirely (`--jitter` then
+has no effect) and `--workers` can go wide; throughput is bound by your bandwidth.
+
+## Deployment
+
+- **Azure** — VM or Container Instances with Blob persistence, plus cost notes.
+  See [`docs/azure.md`](https://github.com/kms2229/pmcfetch/blob/main/docs/azure.md).
+- **HPC / SGE** — `--task-id` (defaulting to `$SGE_TASK_ID`) selects a namespace
+  by 1-based index. See
+  [`examples/sge/download_array.qsub`](https://github.com/kms2229/pmcfetch/blob/main/examples/sge/download_array.qsub).
+
+Nothing in the library is platform-specific; those are examples.
+
+## Corpus formats
+
+`load_records()` normalises keys (`10007932`, `pmc10007932`, `PMC10007932`) and
+accepts:
+
+| Input | Shape |
+|---|---|
+| assignment JSON | `{"<id>": {"metadata": {"namespace": "cardiology"}}}` |
+| full-text dump | `{"<id>": "<full text>"}` |
+| extraction output | `{"pmc10007932": {"sections": [...]}}` |
+| plain array | `["PMC1", "2", ...]` |
+| `.txt` | newline-delimited ids, including pmcfetch's own failure logs |
+
+Keys that aren't PMC ids land in `LoadResult.skipped` rather than raising, so one
+bad key can't abort a 20k-article batch.
+
+## Behaviour worth knowing
+
+- **Resume** checks size *and* `%PDF` magic, so a half-written file from a killed
+  job is re-fetched rather than treated as complete. Downloads are held to the
+  same check before they are written, so every file on disk is one the next run
+  skips.
+- **Writes are atomic** — `.part` + `fsync` + rename; a failed write removes its
+  `.part`.
+- **Retries** cover 429/5xx and transport errors with exponential backoff and full
+  jitter, stretched to any `Retry-After` the server sends (no wait exceeds
+  `backoff_cap`, 15 s by default). 404s and non-PDF bodies are not retried.
+- **Rate limiting** is process-wide, so `--workers` and `--rps` are independent.
+- **Ctrl-C stops promptly** — queued articles are cancelled; only those already
+  in flight finish.
+- **Namespaces become one folder name** — path separators are replaced, so a
+  corpus file cannot write outside `--out`.
+- **Articles with no PDF** raise `PDFNotAvailableError`; ones outside the subset
+  raise `ArticleNotInOASubsetError`. Both land in the failure log with a reason.
+- **Any object implementing `fetch_pdf(pmcid) -> bytes`** satisfies the
+  `PdfSource` protocol and works with `download_many`.
+
+## Tests
+
+```bash
+pip install -e '.[dev]'
+python -m pytest tests/ -q
+```
+
+No network required — the S3 layer is exercised against scripted listings and
+responses: multi-version and paginated listings, supplementary-PDF
+disambiguation, 429 storms and `Retry-After`, 404s, truncated resumes, and
+concurrent and interrupted batches.
+
+## Attribution
+
+Data comes from the
+[NIH NLM NCBI PubMed Central Article Datasets](https://registry.opendata.aws/ncbi-pmc/)
+on AWS Open Data. If you redistribute anything built from it, acknowledge NLM as
+the source and don't imply NLM endorsement — see the
+[bucket README](https://pmc-oa-opendata.s3.amazonaws.com/README.txt).

pmcfetch-0.1.0/README.md

@@ -0,0 +1,156 @@
+# pmcfetch
+
+Targeted PDF retrieval from the **PMC Open Access Subset**, with resume, retries,
+atomic writes, and namespace-aware batch layout. Zero dependencies.
+
+```bash
+pip install pmcfetch
+pmcfetch download --corpus corpus.json --out pdfs/ --workers 16 --rps 0
+```
+
+Requires Python 3.10+. Nothing outside the standard library.
+
+## Why
+
+NLM publishes every PMC Open Access article version in the public bucket
+`s3://pmc-oa-opendata` — anonymous HTTPS, no AWS account, no credentials, no rate
+limit, and the access route PMC sanctions for systematic retrieval.
+
+But `aws s3 sync` is all-or-nothing. There's no `--from-id-list`, so pulling the
+few thousand articles you care about means pulling all ~8 million. And fetching
+them one by one with `aws s3 cp` requires knowing things that aren't obvious:
+
+- **Which version?** Articles live under `PMC10007932.1/`, `.2/`, sometimes `.10/`.
+  You can't know without listing first — and the sort must be numeric, not
+  lexicographic.
+- **Which PDF?** `PMC10007932.1/` also contains `jgo-14-01-302-coif.pdf`,
+  `-dss.pdf`, `-prf.pdf`. Glob `*.pdf` and you get the wrong file.
+- **Not every article has one.** Some licences permit XML and plain-text
+  redistribution but not the PDF.
+
+`pmcfetch` handles all three, then adds the batch machinery you'd otherwise
+rewrite: resume, atomic writes, concurrency, pacing, and structured failure logs.
+
+## Python API
+
+```python
+from pmcfetch import S3OpenAccessClient, load_records, download_many, FailureLog
+
+records = load_records("corpus.json").for_namespace("cardiology")
+
+with S3OpenAccessClient() as client:
+    report = download_many(
+        records, "pdfs/",
+        client=client,
+        workers=16,
+        failure_log=FailureLog("failed.txt"),
+    )
+
+print(report.as_dict())        # {'total': 3139, 'downloaded': 3104, 'skipped': 0, 'failed': 35}
+for r in report.failures:
+    print(r.pmcid, r.reason)
+```
+
+Single article, plus metadata:
+
+```python
+with S3OpenAccessClient() as c:
+    meta = c.metadata("10007932")
+    # {'pmcid': 'PMC10007932', 'doi': '10.21037/jgo-23-76',
+    #  'license_code': 'CC BY-NC-ND', 'is_retracted': False, ...}
+    if not meta["is_retracted"]:
+        pdf = c.fetch_pdf("10007932")   # accepts 10007932 / pmc10007932 / PMC10007932
+```
+
+`license_code` and `is_retracted` come free with every article — filter on licence
+terms and drop retracted papers before they reach your corpus.
+
+## CLI
+
+```bash
+# what's in a corpus file?
+pmcfetch namespaces --corpus corpus.json
+
+# pull one namespace
+pmcfetch download --corpus corpus.json --out pdfs/ --namespace cardiology --workers 16 --rps 0
+
+# dry run, no requests
+pmcfetch download --corpus corpus.json --out pdfs/ --limit 20 --dry-run
+
+# retry failures — the log doubles as an input list
+pmcfetch download --corpus pdfs/failed.txt --out pdfs/ --flat
+```
+
+Exit code is `1` when any article failed, `0` otherwise — so a batch job fails
+loudly instead of silently producing a short corpus. Bad arguments or an
+unreadable corpus file exit `2`, and Ctrl-C exits `130`.
+
+S3 has no rate limit, so `--rps 0` switches pacing off entirely (`--jitter` then
+has no effect) and `--workers` can go wide; throughput is bound by your bandwidth.
+
+## Deployment
+
+- **Azure** — VM or Container Instances with Blob persistence, plus cost notes.
+  See [`docs/azure.md`](https://github.com/kms2229/pmcfetch/blob/main/docs/azure.md).
+- **HPC / SGE** — `--task-id` (defaulting to `$SGE_TASK_ID`) selects a namespace
+  by 1-based index. See
+  [`examples/sge/download_array.qsub`](https://github.com/kms2229/pmcfetch/blob/main/examples/sge/download_array.qsub).
+
+Nothing in the library is platform-specific; those are examples.
+
+## Corpus formats
+
+`load_records()` normalises keys (`10007932`, `pmc10007932`, `PMC10007932`) and
+accepts:
+
+| Input | Shape |
+|---|---|
+| assignment JSON | `{"<id>": {"metadata": {"namespace": "cardiology"}}}` |
+| full-text dump | `{"<id>": "<full text>"}` |
+| extraction output | `{"pmc10007932": {"sections": [...]}}` |
+| plain array | `["PMC1", "2", ...]` |
+| `.txt` | newline-delimited ids, including pmcfetch's own failure logs |
+
+Keys that aren't PMC ids land in `LoadResult.skipped` rather than raising, so one
+bad key can't abort a 20k-article batch.
+
+## Behaviour worth knowing
+
+- **Resume** checks size *and* `%PDF` magic, so a half-written file from a killed
+  job is re-fetched rather than treated as complete. Downloads are held to the
+  same check before they are written, so every file on disk is one the next run
+  skips.
+- **Writes are atomic** — `.part` + `fsync` + rename; a failed write removes its
+  `.part`.
+- **Retries** cover 429/5xx and transport errors with exponential backoff and full
+  jitter, stretched to any `Retry-After` the server sends (no wait exceeds
+  `backoff_cap`, 15 s by default). 404s and non-PDF bodies are not retried.
+- **Rate limiting** is process-wide, so `--workers` and `--rps` are independent.
+- **Ctrl-C stops promptly** — queued articles are cancelled; only those already
+  in flight finish.
+- **Namespaces become one folder name** — path separators are replaced, so a
+  corpus file cannot write outside `--out`.
+- **Articles with no PDF** raise `PDFNotAvailableError`; ones outside the subset
+  raise `ArticleNotInOASubsetError`. Both land in the failure log with a reason.
+- **Any object implementing `fetch_pdf(pmcid) -> bytes`** satisfies the
+  `PdfSource` protocol and works with `download_many`.
+
+## Tests
+
+```bash
+pip install -e '.[dev]'
+python -m pytest tests/ -q
+```
+
+No network required — the S3 layer is exercised against scripted listings and
+responses: multi-version and paginated listings, supplementary-PDF
+disambiguation, 429 storms and `Retry-After`, 404s, truncated resumes, and
+concurrent and interrupted batches.
+
+## Attribution
+
+Data comes from the
+[NIH NLM NCBI PubMed Central Article Datasets](https://registry.opendata.aws/ncbi-pmc/)
+on AWS Open Data. If you redistribute anything built from it, acknowledge NLM as
+the source and don't imply NLM endorsement — see the
+[bucket README](https://pmc-oa-opendata.s3.amazonaws.com/README.txt).

pmcfetch-0.1.0/pyproject.toml

@@ -0,0 +1,61 @@
+[build-system]
+requires = ["setuptools>=77", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pmcfetch"
+dynamic = ["version"]
+description = "Targeted PDF retrieval from the PMC Open Access Subset on AWS. Zero dependencies."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Kavach Shah" }]
+requires-python = ">=3.10"
+keywords = ["pubmed-central", "pmc", "open-access", "bulk-download", "bioinformatics"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Environment :: Console",
+  "Intended Audience :: Science/Research",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+  "Topic :: Scientific/Engineering :: Bio-Informatics",
+  "Topic :: Text Processing :: Indexing",
+  "Typing :: Typed",
+]
+# Stdlib only — S3 Open Access is anonymous HTTPS.
+dependencies = []
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/kms2229/pmcfetch"
+Issues = "https://github.com/kms2229/pmcfetch/issues"
+
+[project.scripts]
+pmcfetch = "pmcfetch.cli:main"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+pmcfetch = ["py.typed"]
+
+[tool.setuptools.dynamic]
+# Single source of truth: pmcfetch.__version__ (also used in the User-Agent).
+version = { attr = "pmcfetch.__version__" }
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = ["-q"]

pmcfetch-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pmcfetch-0.1.0/src/pmcfetch/__init__.py

@@ -0,0 +1,102 @@
+"""Targeted PDF retrieval from the PMC Open Access Subset.
+
+NLM publishes every PMC Open Access article version in the public, anonymous
+bucket ``s3://pmc-oa-opendata``. No AWS account, no credentials, no rate limit,
+and it is the access route PMC sanctions for systematic retrieval.
+
+``aws s3 sync`` is all-or-nothing — there is no way to pull just the articles you
+care about. This package fills that gap: give it a list of PMC ids and it
+resolves each article's version, picks the article PDF (not the publisher's
+supplementary PDFs sitting in the same prefix), and writes the results with
+resume, retries, atomic writes, and per-namespace layout.
+
+Zero third-party dependencies.
+
+    from pmcfetch import S3OpenAccessClient, load_records, download_many
+
+    records = load_records("corpus.json").for_namespace("cardiology")
+    with S3OpenAccessClient() as client:
+        report = download_many(records, "pdfs/", client=client, workers=16)
+    print(report.as_dict())
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from pmcfetch.batch import (
+    BatchReport,
+    DownloadResult,
+    FailureLog,
+    Outcome,
+    download_many,
+    download_one,
+    target_path,
+    write_pdf_atomic,
+)
+from pmcfetch.errors import (
+    DownloadError,
+    HTTPStatusError,
+    InvalidPMCIDError,
+    NotAPDFError,
+    PMCFetchError,
+    TransportError,
+)
+from pmcfetch.ids import normalize_pmcid, try_normalize_pmcid
+from pmcfetch.manifest import CorpusRecord, LoadResult, iter_records, load_records
+from pmcfetch.pdf import DEFAULT_MIN_PDF_BYTES, DEFAULT_TIMEOUT, PdfSource, is_pdf
+from pmcfetch.ratelimit import MinIntervalLimiter, NullLimiter
+from pmcfetch.s3source import (
+    OA_BASE_URL,
+    OA_BUCKET,
+    ArticleNotInOASubsetError,
+    ArticleVersion,
+    PDFNotAvailableError,
+    S3OpenAccessClient,
+    list_article_versions,
+    resolve_latest_version,
+)
+
+__all__ = [
+    "__version__",
+    # source
+    "S3OpenAccessClient",
+    "OA_BUCKET",
+    "OA_BASE_URL",
+    "ArticleVersion",
+    "list_article_versions",
+    "resolve_latest_version",
+    # batch
+    "download_one",
+    "download_many",
+    "BatchReport",
+    "DownloadResult",
+    "Outcome",
+    "FailureLog",
+    "target_path",
+    "write_pdf_atomic",
+    # manifest
+    "load_records",
+    "iter_records",
+    "CorpusRecord",
+    "LoadResult",
+    # ids
+    "normalize_pmcid",
+    "try_normalize_pmcid",
+    # pdf / pacing
+    "PdfSource",
+    "is_pdf",
+    "DEFAULT_MIN_PDF_BYTES",
+    "DEFAULT_TIMEOUT",
+    "MinIntervalLimiter",
+    "NullLimiter",
+    # errors
+    "PMCFetchError",
+    "DownloadError",
+    "HTTPStatusError",
+    "NotAPDFError",
+    "PDFNotAvailableError",
+    "ArticleNotInOASubsetError",
+    "InvalidPMCIDError",
+    "TransportError",
+]