pmc-toolkit 0.1.0__tar.gz

pmc_toolkit-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,44 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint-typecheck-test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --all-extras --dev --python ${{ matrix.python-version }}
+
+      - name: Ruff lint
+        run: uv run ruff check .
+
+      - name: Ruff format check
+        run: uv run ruff format --check .
+
+      - name: Type check (ty)
+        run: uv run ty check
+
+      - name: Pytest
+        run: uv run pytest

pmc_toolkit-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,41 @@
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pmc-toolkit
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
+      - name: Set up Python
+        run: uv python install 3.13
+
+      - name: Verify tag matches package version
+        run: |
+          tag="${GITHUB_REF#refs/tags/v}"
+          pkg=$(uv version --short)
+          if [ "$tag" != "$pkg" ]; then
+            echo "::error::Git tag v$tag does not match pyproject.toml version $pkg"
+            exit 1
+          fi
+
+      - name: Build distributions
+        run: uv build --no-sources
+
+      - name: Publish to PyPI (Trusted Publishing)
+        run: uv publish

pmc_toolkit-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

pmc_toolkit-0.1.0/.python-version

@@ -0,0 +1 @@
+3.14

pmc_toolkit-0.1.0/AGENTS.md

@@ -0,0 +1,19 @@
+# Agent / dev notes
+
+- ALWAYS use **uv** for Python environment, dependency, and tool commands; do not use **pip**, **python -m pip**, **virtualenv**, **poetry**, or similar unless explicitly asked.
+
+- ALWAYS run repo tools via **`uv run`** from the project root.
+
+- AFTER Python edits, run **`uv run ty check`**, **`uv run ruff check`**, and **`uv run ruff format`**; run **`uv sync`** when dependencies change.
+
+- AFTER behavior changes, run **`uv run pytest`** to check for regressions and report the result.
+
+- PREFER **`uv run pytest`** with a path, node id, or **`-k`** while iterating, not the whole suite each time.
+
+- NEVER start development servers, watchers, builds, or long-running local processes unless explicitly asked.
+
+- NEVER add or modify tests unless explicitly asked.
+
+- NEVER assume **`ruff`**, **`ty check`**, or **`pytest`** failures on main are pre-existing.
+
+- AVOID shortened names; prefer descriptive names like `version` over `ver`.

pmc_toolkit-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jaka
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pmc_toolkit-0.1.0/PKG-INFO

@@ -0,0 +1,183 @@
+Metadata-Version: 2.4
+Name: pmc-toolkit
+Version: 0.1.0
+Summary: Python toolkit and CLI for exploring, downloading, and parsing PMC article data.
+Project-URL: Homepage, https://github.com/JakaKokosar/pmc-toolkit
+Project-URL: Repository, https://github.com/JakaKokosar/pmc-toolkit
+Project-URL: Issues, https://github.com/JakaKokosar/pmc-toolkit/issues
+Author: Jaka Kokosar
+License-Expression: MIT
+License-File: LICENSE
+Keywords: bioinformatics,full-text,ncbi,open-access,pmc,pubmed
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Classifier: Topic :: Text Processing :: Markup :: XML
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: boto3>=1.42.93
+Requires-Dist: lxml>=6.1.0
+Requires-Dist: platformdirs>=4.3.0
+Requires-Dist: pydantic>=2.13.3
+Requires-Dist: typer>=0.24.1
+Description-Content-Type: text/markdown
+
+# PMC Toolkit
+
+Python toolkit and CLI for exploring, downloading, and parsing PMC article data
+from the PMC Open Access dataset on AWS S3 (`s3://pmc-oa-opendata`).
+
+## Current Status
+
+The project currently supports:
+
+- listing available versions for a PMCID
+- validating PMC identifiers before making requests
+- retrieving metadata for a PMC identifier, defaulting to the latest version for a base PMCID
+- listing every object for a resolved article version, using the local cache when available
+- downloading files for an article version into a local cache (optional `--ext`
+  filters apply only to `fetch`, not to `files`; `--ext` accepts either a
+  comma-separated list or repeated flags)
+- parsing cached full-text XML into a normalized article dictionary with
+  title, journal, article, affiliations, author notes, abstract, content,
+  acknowledgements, data availability, related articles, custom metadata,
+  competing interests, supplementary media, references, figures, and tables
+
+## Requirements
+
+- Python 3.11+
+- `uv`
+
+## Setup
+
+```bash
+uv sync
+```
+
+## Development
+
+After code changes, run the checks in [AGENTS.md](AGENTS.md) (typecheck, Ruff, tests).
+
+## CLI Usage
+
+Show the available commands:
+
+```bash
+uv run pmc --help
+```
+
+CLI commands print indented JSON to stdout.
+
+List versions for a PMC article:
+
+```bash
+uv run pmc versions PMC11370360
+```
+
+Fetch metadata for the latest available version of a PMCID:
+
+```bash
+uv run pmc metadata PMC11370360
+```
+
+Fetch metadata for a specific version:
+
+```bash
+uv run pmc metadata PMC11370360.1
+```
+
+List every object key for an article version (including media and supplements).
+For unversioned IDs, the CLI resolves the latest version from S3 first; once the
+version is known, the cached object-key manifest is reused when present. There
+is no extension filter on this command.
+
+```bash
+uv run pmc files PMC11370360.1
+```
+
+Download files to a local cache. The default root is the **per-OS user cache
+directory** from
+[`platformdirs`](https://github.com/tox-dev/platformdirs) (e.g. `~/.cache/pmc-toolkit` on
+Linux, `~/Library/Caches/pmc-toolkit` on macOS, and under `%LOCALAPPDATA%` on
+Windows), with files under `<root>/<PMCid.N>/`. Override with `--cache-dir` or
+`PMC_TOOLKIT_CACHE`.
+
+```bash
+uv run pmc fetch PMC11370360.1
+```
+
+Download only selected file types, re-downloading even if cached:
+
+```bash
+uv run pmc fetch PMC11370360.1 --ext xml,pdf,jpg --force
+```
+
+The `--ext` option also accepts repeated flags if you prefer the more explicit
+form:
+
+```bash
+uv run pmc fetch PMC11370360.1 --ext pdf --ext xml --ext jpg --force
+```
+
+Override the cache location via a flag or the `PMC_TOOLKIT_CACHE` env var:
+
+```bash
+uv run pmc fetch PMC11370360.1 --cache-dir ./data
+PMC_TOOLKIT_CACHE=./data uv run pmc fetch PMC11370360.1
+```
+
+Convert a cached XML file into extracted JSON. Run `fetch --ext xml` first if
+the XML is not already in the cache. The first conversion parses XML once,
+writes `<cache-root>/<PMCid.N>/.pmc-extracted-article.json`, and prints the
+extracted JSON; later conversions for the same article version read that JSON
+cache unless `--force` is passed.
+
+```bash
+uv run pmc fetch PMC11370360.1 --ext xml
+uv run pmc convert-xml PMC11370360.1
+```
+
+List the extracted JSON top-level keys:
+
+```bash
+uv run pmc convert-xml --list-keys PMC11370360.1
+```
+
+`article_info.publication_date` currently uses the first publication date found
+in the XML. If downstream consumers need to distinguish date types such as
+`epub`, `ppub`, or `collection`, the output can be extended later.
+
+## Project Layout
+
+Here **“storage”** means the AWS bucket plus the local cache directory where
+`pmc fetch` writes files—not a database or ORM.
+
+- `src/pmc_toolkit/cli.py` - Typer CLI commands
+- `src/pmc_toolkit/storage_api.py` - import this for programmatic use: list versions, metadata, list all keys, fetch to cache
+- `src/pmc_toolkit/storage_utils.py` - boto3/unsigned S3 client, list-objects, downloads; implementation details for `storage_api`
+- `src/pmc_toolkit/xml_parse_api.py` - import this for programmatic parsing of cached XML files
+- `src/pmc_toolkit/xml_parse_utils.py` - small `lxml`-based helpers for cached full-text XML extraction
+- `src/pmc_toolkit/cache.py` - per-article directories under the cache root, JSON metadata, cached S3 key listings, and safe local paths for downloaded objects
+- `src/pmc_toolkit/validators.py` - identifier validation
+- `src/pmc_toolkit/models.py` - response models
+- `tests/` - automated tests
+
+### Local cache
+
+Each resolved article version has a directory `<cache_root>/<PMCid.N>/` containing:
+
+- **`<PMCid.N>.json`** — cached metadata (from S3 `metadata/<PMCid.N>.json`), written after a successful read.
+- **`.pmc-object-keys.json`** — JSON array of S3 object keys under that article’s prefix, written after `list_objects_v2` (or read on cache hit). If this file is missing or not a list of strings, listing or fetch may refetch from S3 or raise `ValueError` for an invalid manifest.
+- **`.pmc-extracted-article.json`** — full extracted JSON produced from the cached XML by `pmc convert-xml`; reused by later conversions for the same article version.
+
+**Cache root selection:** `pmc metadata` and `pmc files` (and the matching `storage_api` functions) always use the default OS user cache from [`platformdirs`](https://github.com/tox-dev/platformdirs). Only `pmc fetch` and `fetch_files(..., cache_dir=...)` accept `--cache-dir` or the `PMC_TOOLKIT_CACHE` environment variable.
+
+**Download paths:** For each S3 key, the toolkit maps `PMCid.N/relative/path` to `<cache_root>/PMCid.N/relative/path`. Keys that do not start with the `PMCid.N/` prefix, use an absolute path segment, or resolve outside that directory (for example `..` path segments) are rejected with `ValueError` so downloads never leave the article folder.

pmc_toolkit-0.1.0/README.md

@@ -0,0 +1,151 @@
+# PMC Toolkit
+
+Python toolkit and CLI for exploring, downloading, and parsing PMC article data
+from the PMC Open Access dataset on AWS S3 (`s3://pmc-oa-opendata`).
+
+## Current Status
+
+The project currently supports:
+
+- listing available versions for a PMCID
+- validating PMC identifiers before making requests
+- retrieving metadata for a PMC identifier, defaulting to the latest version for a base PMCID
+- listing every object for a resolved article version, using the local cache when available
+- downloading files for an article version into a local cache (optional `--ext`
+  filters apply only to `fetch`, not to `files`; `--ext` accepts either a
+  comma-separated list or repeated flags)
+- parsing cached full-text XML into a normalized article dictionary with
+  title, journal, article, affiliations, author notes, abstract, content,
+  acknowledgements, data availability, related articles, custom metadata,
+  competing interests, supplementary media, references, figures, and tables
+
+## Requirements
+
+- Python 3.11+
+- `uv`
+
+## Setup
+
+```bash
+uv sync
+```
+
+## Development
+
+After code changes, run the checks in [AGENTS.md](AGENTS.md) (typecheck, Ruff, tests).
+
+## CLI Usage
+
+Show the available commands:
+
+```bash
+uv run pmc --help
+```
+
+CLI commands print indented JSON to stdout.
+
+List versions for a PMC article:
+
+```bash
+uv run pmc versions PMC11370360
+```
+
+Fetch metadata for the latest available version of a PMCID:
+
+```bash
+uv run pmc metadata PMC11370360
+```
+
+Fetch metadata for a specific version:
+
+```bash
+uv run pmc metadata PMC11370360.1
+```
+
+List every object key for an article version (including media and supplements).
+For unversioned IDs, the CLI resolves the latest version from S3 first; once the
+version is known, the cached object-key manifest is reused when present. There
+is no extension filter on this command.
+
+```bash
+uv run pmc files PMC11370360.1
+```
+
+Download files to a local cache. The default root is the **per-OS user cache
+directory** from
+[`platformdirs`](https://github.com/tox-dev/platformdirs) (e.g. `~/.cache/pmc-toolkit` on
+Linux, `~/Library/Caches/pmc-toolkit` on macOS, and under `%LOCALAPPDATA%` on
+Windows), with files under `<root>/<PMCid.N>/`. Override with `--cache-dir` or
+`PMC_TOOLKIT_CACHE`.
+
+```bash
+uv run pmc fetch PMC11370360.1
+```
+
+Download only selected file types, re-downloading even if cached:
+
+```bash
+uv run pmc fetch PMC11370360.1 --ext xml,pdf,jpg --force
+```
+
+The `--ext` option also accepts repeated flags if you prefer the more explicit
+form:
+
+```bash
+uv run pmc fetch PMC11370360.1 --ext pdf --ext xml --ext jpg --force
+```
+
+Override the cache location via a flag or the `PMC_TOOLKIT_CACHE` env var:
+
+```bash
+uv run pmc fetch PMC11370360.1 --cache-dir ./data
+PMC_TOOLKIT_CACHE=./data uv run pmc fetch PMC11370360.1
+```
+
+Convert a cached XML file into extracted JSON. Run `fetch --ext xml` first if
+the XML is not already in the cache. The first conversion parses XML once,
+writes `<cache-root>/<PMCid.N>/.pmc-extracted-article.json`, and prints the
+extracted JSON; later conversions for the same article version read that JSON
+cache unless `--force` is passed.
+
+```bash
+uv run pmc fetch PMC11370360.1 --ext xml
+uv run pmc convert-xml PMC11370360.1
+```
+
+List the extracted JSON top-level keys:
+
+```bash
+uv run pmc convert-xml --list-keys PMC11370360.1
+```
+
+`article_info.publication_date` currently uses the first publication date found
+in the XML. If downstream consumers need to distinguish date types such as
+`epub`, `ppub`, or `collection`, the output can be extended later.
+
+## Project Layout
+
+Here **“storage”** means the AWS bucket plus the local cache directory where
+`pmc fetch` writes files—not a database or ORM.
+
+- `src/pmc_toolkit/cli.py` - Typer CLI commands
+- `src/pmc_toolkit/storage_api.py` - import this for programmatic use: list versions, metadata, list all keys, fetch to cache
+- `src/pmc_toolkit/storage_utils.py` - boto3/unsigned S3 client, list-objects, downloads; implementation details for `storage_api`
+- `src/pmc_toolkit/xml_parse_api.py` - import this for programmatic parsing of cached XML files
+- `src/pmc_toolkit/xml_parse_utils.py` - small `lxml`-based helpers for cached full-text XML extraction
+- `src/pmc_toolkit/cache.py` - per-article directories under the cache root, JSON metadata, cached S3 key listings, and safe local paths for downloaded objects
+- `src/pmc_toolkit/validators.py` - identifier validation
+- `src/pmc_toolkit/models.py` - response models
+- `tests/` - automated tests
+
+### Local cache
+
+Each resolved article version has a directory `<cache_root>/<PMCid.N>/` containing:
+
+- **`<PMCid.N>.json`** — cached metadata (from S3 `metadata/<PMCid.N>.json`), written after a successful read.
+- **`.pmc-object-keys.json`** — JSON array of S3 object keys under that article’s prefix, written after `list_objects_v2` (or read on cache hit). If this file is missing or not a list of strings, listing or fetch may refetch from S3 or raise `ValueError` for an invalid manifest.
+- **`.pmc-extracted-article.json`** — full extracted JSON produced from the cached XML by `pmc convert-xml`; reused by later conversions for the same article version.
+
+**Cache root selection:** `pmc metadata` and `pmc files` (and the matching `storage_api` functions) always use the default OS user cache from [`platformdirs`](https://github.com/tox-dev/platformdirs). Only `pmc fetch` and `fetch_files(..., cache_dir=...)` accept `--cache-dir` or the `PMC_TOOLKIT_CACHE` environment variable.
+
+**Download paths:** For each S3 key, the toolkit maps `PMCid.N/relative/path` to `<cache_root>/PMCid.N/relative/path`. Keys that do not start with the `PMCid.N/` prefix, use an absolute path segment, or resolve outside that directory (for example `..` path segments) are rejected with `ValueError` so downloads never leave the article folder.

pmc_toolkit-0.1.0/RELEASING.md

@@ -0,0 +1,44 @@
+# Releasing
+
+Pushing a `v*` git tag triggers [`release.yml`](.github/workflows/release.yml),
+which builds with `uv build` and publishes to
+[PyPI](https://pypi.org/project/pmc-toolkit/) via Trusted Publishing (OIDC,
+no tokens).
+
+## Release flow
+
+```sh
+# 1. Make sure main is green and up to date.
+git switch main && git pull
+
+# 2. Bump version (patch | minor | major, or X.Y.Z for an exact version).
+uv version --bump patch
+version="$(uv version --short)"
+
+# 3. Commit, push, wait for CI to go green.
+git add pyproject.toml uv.lock
+git commit -m "release: v${version}"
+git push
+
+# 4. Tag and push — this triggers the publish.
+git tag "v${version}"
+git push origin "v${version}"
+```
+
+Watch the **Release** workflow in the Actions tab. Approve the `pypi`
+deployment if the environment requires it. Smoke test:
+
+```sh
+uv run --with "pmc-toolkit==${version}" --no-project -- pmc --help
+```
+
+Optionally draft a GitHub Release from the tag for user-facing notes.
+
+## Troubleshooting
+
+- **`invalid-publisher`** — PyPI trusted-publisher fields don't match the
+  workflow run (case-sensitive).
+- **Tag/version mismatch** — fix `pyproject.toml` or delete the bad tag:
+  `git tag -d vX.Y.Z && git push --delete origin vX.Y.Z`.
+- **Bad release** — PyPI forbids re-uploading the same version. Yank it on
+  PyPI, bump again, re-release.

pmc_toolkit-0.1.0/pyproject.toml

@@ -0,0 +1,55 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "pmc-toolkit"
+version = "0.1.0"
+description = "Python toolkit and CLI for exploring, downloading, and parsing PMC article data."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Jaka Kokosar" }]
+keywords = ["pmc", "pubmed", "bioinformatics", "ncbi", "open-access", "full-text"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Bio-Informatics",
+    "Topic :: Text Processing :: Markup :: XML",
+    "Typing :: Typed",
+]
+dependencies = [
+    "boto3>=1.42.93",
+    "lxml>=6.1.0",
+    "platformdirs>=4.3.0",
+    "pydantic>=2.13.3",
+    "typer>=0.24.1",
+]
+
+[project.urls]
+Homepage = "https://github.com/JakaKokosar/pmc-toolkit"
+Repository = "https://github.com/JakaKokosar/pmc-toolkit"
+Issues = "https://github.com/JakaKokosar/pmc-toolkit/issues"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.3",
+    "ruff>=0.15.11",
+    "ty>=0.0.32",
+    "types-boto3[s3]>=1.42.94",
+]
+
+[project.scripts]
+pmc = "pmc_toolkit.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/pmc_toolkit"]