pmc-toolkit 0.2.0__tar.gz → 0.3.0__tar.gz

{pmc_toolkit-0.2.0 → pmc_toolkit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pmc-toolkit
-Version: 0.2.0
+Version: 0.3.0
 Summary: Python toolkit and CLI for exploring, downloading, and parsing PMC article data.
 Project-URL: Homepage, https://github.com/JakaKokosar/pmc-toolkit
 Project-URL: Repository, https://github.com/JakaKokosar/pmc-toolkit
@@ -134,22 +134,17 @@ uv run pmc-toolkit fetch PMC11370360.1 --cache-dir ./data
 PMC_TOOLKIT_CACHE=./data uv run pmc-toolkit fetch PMC11370360.1
 ```
 
-Convert a cached XML file into extracted JSON. Run `fetch --ext xml` first if
-the XML is not already in the cache. The first conversion parses XML once,
+Parse a cached XML file into extracted JSON. Run `fetch --ext xml` first if
+the XML is not already in the cache. The first parse reads XML once,
 writes `<cache-root>/<PMCid.N>/.pmc-extracted-article.json`, and prints the
-extracted JSON; later conversions for the same article version read that JSON
+extracted JSON; later parses for the same article version read that JSON
 cache unless `--force` is passed.
 
 ```bash
 uv run pmc-toolkit fetch PMC11370360.1 --ext xml
-uv run pmc-toolkit convert-xml PMC11370360.1
+uv run pmc-toolkit parse PMC11370360.1
 ```
 
-List the extracted JSON top-level keys:
-
-```bash
-uv run pmc-toolkit convert-xml --list-keys PMC11370360.1
-```
 
 `article_info.publication_date` currently uses the first publication date found
 in the XML. If downstream consumers need to distinguish date types such as
@@ -176,7 +171,7 @@ Each resolved article version has a directory `<cache_root>/<PMCid.N>/` containi
 
 - **`<PMCid.N>.json`** — cached metadata (from S3 `metadata/<PMCid.N>.json`), written after a successful read.
 - **`.pmc-object-keys.json`** — JSON array of S3 object keys under that article’s prefix, written after `list_objects_v2` (or read on cache hit). If this file is missing or not a list of strings, listing or fetch may refetch from S3 or raise `ValueError` for an invalid manifest.
-- **`.pmc-extracted-article.json`** — full extracted JSON produced from the cached XML by `pmc-toolkit convert-xml`; reused by later conversions for the same article version.
+- **`.pmc-extracted-article.json`** — full extracted JSON produced from the cached XML by `pmc-toolkit parse`; reused by later parses for the same article version.
 
 **Cache root selection:** `pmc-toolkit metadata` and `pmc-toolkit files` (and the matching `storage_api` functions) always use the default OS user cache from [`platformdirs`](https://github.com/tox-dev/platformdirs). Only `pmc-toolkit fetch` and `fetch_files(..., cache_dir=...)` accept `--cache-dir` or the `PMC_TOOLKIT_CACHE` environment variable.
 

{pmc_toolkit-0.2.0 → pmc_toolkit-0.3.0}/README.md

@@ -102,22 +102,17 @@ uv run pmc-toolkit fetch PMC11370360.1 --cache-dir ./data
 PMC_TOOLKIT_CACHE=./data uv run pmc-toolkit fetch PMC11370360.1
 ```
 
-Convert a cached XML file into extracted JSON. Run `fetch --ext xml` first if
-the XML is not already in the cache. The first conversion parses XML once,
+Parse a cached XML file into extracted JSON. Run `fetch --ext xml` first if
+the XML is not already in the cache. The first parse reads XML once,
 writes `<cache-root>/<PMCid.N>/.pmc-extracted-article.json`, and prints the
-extracted JSON; later conversions for the same article version read that JSON
+extracted JSON; later parses for the same article version read that JSON
 cache unless `--force` is passed.
 
 ```bash
 uv run pmc-toolkit fetch PMC11370360.1 --ext xml
-uv run pmc-toolkit convert-xml PMC11370360.1
+uv run pmc-toolkit parse PMC11370360.1
 ```
 
-List the extracted JSON top-level keys:
-
-```bash
-uv run pmc-toolkit convert-xml --list-keys PMC11370360.1
-```
 
 `article_info.publication_date` currently uses the first publication date found
 in the XML. If downstream consumers need to distinguish date types such as
@@ -144,7 +139,7 @@ Each resolved article version has a directory `<cache_root>/<PMCid.N>/` containi
 
 - **`<PMCid.N>.json`** — cached metadata (from S3 `metadata/<PMCid.N>.json`), written after a successful read.
 - **`.pmc-object-keys.json`** — JSON array of S3 object keys under that article’s prefix, written after `list_objects_v2` (or read on cache hit). If this file is missing or not a list of strings, listing or fetch may refetch from S3 or raise `ValueError` for an invalid manifest.
-- **`.pmc-extracted-article.json`** — full extracted JSON produced from the cached XML by `pmc-toolkit convert-xml`; reused by later conversions for the same article version.
+- **`.pmc-extracted-article.json`** — full extracted JSON produced from the cached XML by `pmc-toolkit parse`; reused by later parses for the same article version.
 
 **Cache root selection:** `pmc-toolkit metadata` and `pmc-toolkit files` (and the matching `storage_api` functions) always use the default OS user cache from [`platformdirs`](https://github.com/tox-dev/platformdirs). Only `pmc-toolkit fetch` and `fetch_files(..., cache_dir=...)` accept `--cache-dir` or the `PMC_TOOLKIT_CACHE` environment variable.
 

{pmc_toolkit-0.2.0 → pmc_toolkit-0.3.0}/RELEASING.md

@@ -32,6 +32,10 @@ deployment if the environment requires it. Smoke test:
 uv run --with "pmc-toolkit==${version}" --no-project -- pmc-toolkit --help
 ```
 
+From **v0.2.0**, the PyPI wheel exposes only the `pmc-toolkit` console script
+(the previous `pmc` script was removed so the binary matches the distribution
+name).
+
 Optionally draft a GitHub Release from the tag for user-facing notes.
 
 ## Troubleshooting

{pmc_toolkit-0.2.0 → pmc_toolkit-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pmc-toolkit"
-version = "0.2.0"
+version = "0.3.0"
 description = "Python toolkit and CLI for exploring, downloading, and parsing PMC article data."
 readme = "README.md"
 requires-python = ">=3.11"

pmc_toolkit-0.3.0/skills/pmc-toolkit/SKILL.md

@@ -0,0 +1,50 @@
+---
+name: pmc-toolkit
+description: Retrieve PubMed Central (PMC) Open Access article data from a PMCID or versioned PMCID, including version discovery, metadata, S3 file listings, downloads, cached XML parsing, structured article JSON, references, figures, tables, affiliations, author notes, funding, acknowledgements, data availability, competing interests, supplementary media, related articles, or custom metadata. Covers PMC Toolkit CLI and Python API workflows. Not for PubMed-only PMID lookup or general literature search without PMC OA article retrieval.
+---
+
+# PMC Toolkit
+
+Use this skill to retrieve, download, and parse PMC Open Access article data with PMC Toolkit. Select commands by the data needed to complete the task, not by surface wording in the request.
+
+## Operating Rules
+
+- Run published-tool commands as `uvx pmc-toolkit ...`.
+- Do not add installation guidance. If `uv` or `uvx` is unavailable, report that PMC Toolkit needs it and stop.
+- Live lookups, listings, and downloads require network access to the PMC Open Access S3 dataset unless the needed data is already cached.
+- Prefer targeted XML, metadata, and media retrieval over whole-PDF parsing. Use PDFs only when the user asks or just need to download for other reasons.
+
+## Quick overview
+
+1. Treat a base PMCID such as `<PMCID>` as the normal input. Convert it to an explicit versioned PMCID once, then reuse `<PMCID.N>` for downstream commands.
+2. `versions <PMCID>` lists every published `<PMCID.N>`. Use it to enumerate versions or pin a concrete version for scripts (for example latest via `jq -r '.versions[-1]'`). Details: [references/cli-versions.md](references/cli-versions.md).
+3. `metadata <PMCID>` or `metadata <PMCID.N>` returns bibliographic fields, OA flags, and S3 URL fields (`xml_url`, `pdf_url`, and so on). It is **not** the primary command for **resolving which `<PMCID.N>` exists or picking a version**. Details: [references/cli-metadata.md](references/cli-metadata.md).
+4. `files <PMCID.N>` returns the complete S3 object-key inventory for that article version. It is the discovery step for available XML, PDFs, figures, media, and supplements.
+5. `fetch <PMCID.N>` downloads all or filtered S3 objects into the local article cache. Add `--ext` for specific file types, `--cache-dir` for a task-specific cache, and `--force` to refresh cached files.
+6. `parse <PMCID.N>` transforms cached full-text XML into normalized article JSON with top-level keys `article_info`, `content`, `references`, `figures`, `tables`, and `supporting_info`. Use it after XML is present in the selected cache; add `--force` to rebuild the extracted JSON cache. Details: [references/cli-parse.md](references/cli-parse.md).
+
+## Context-first retrieval
+
+Choose the smallest useful retrieval path for the question:
+
+- Metadata, DOI, license, version, OA status, or file availability: run `versions` if a pinned version matters, then `metadata` or `files`.
+- Abstract title, content sections, funding grants, authors, affiliations, references, figure captions, tables, or supporting statements, use `parse` to get the article JSON:
+    - Figure questions: inspect `figures[]` from extracted JSON first; fetch only the referenced image extensions when the visual itself is needed.
+    - Table questions: inspect `tables[]`; if it is empty, report that no structured XML tables were found rather than falling back to PDF parsing automatically.
+    - Supplement, data availability, acknowledgements, competing interests, author contribution, or correspondence questions: inspect `supporting_info`.
+    - Citation and evidence grounding: use link-aware paragraph fields (`source_id`, `reference_ids`, `figure_ids`, and `table_ids`) to retrieve only linked references, figures, or tables needed for the answer.
+
+## Additional resources
+
+- [references/cli-versions.md](references/cli-versions.md) — Examples for `versions`, including selecting the latest or a specific `<PMCID.N>`.
+- [references/cli-metadata.md](references/cli-metadata.md) — Examples and field overview for `metadata`.
+- [references/cli-parse.md](references/cli-parse.md) — Examples and field overview for `parse` output.
+
+## Gotchas
+
+- `versions` rejects versioned IDs; pass only a base PMCID.
+- `metadata`, `files`, `fetch`, and `parse` accept base or versioned IDs. Passing a base ID makes those commands resolve the latest version at run time. Prefer `versions <PMCID>` (then reuse the chosen `<PMCID.N>`) when the work needs an explicit pinned version rather than repeating implicit latest-resolution on each command.
+- `files` has no extension filter. Use `fetch --ext` for filtered downloads.
+- `parse` needs `<cache>/<PMCID.N>/<PMCID.N>.xml`; run `fetch <PMCID.N> --ext xml` first when the XML is absent.
+- `metadata` and `files` use the default OS user cache. `fetch` and `parse` can use `--cache-dir` or `PMC_TOOLKIT_CACHE`.
+- Cache paths are per article version. Keep the same cache root across `fetch` and `parse`.

pmc_toolkit-0.3.0/skills/pmc-toolkit/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "PMC Toolkit"
+  short_description: "Targeted PMC OA paper retrieval"
+  default_prompt: "Use $pmc-toolkit to retrieve only the PMC paper metadata, text sections, figures, tables, or supporting info needed for this question."

pmc_toolkit-0.3.0/skills/pmc-toolkit/references/cli-metadata.md

@@ -0,0 +1,36 @@
+# CLI: `metadata`
+
+Use `metadata <PMCID>` or `metadata <PMCID.N>` to fetch bibliographic fields, Open Access flags, and S3 URL fields (for example `xml_url`, `pdf_url`, `media_urls`, `text_url`), plus `pmid` and `doi`.
+
+**Version resolution:** When the task is to discover which `<PMCID.N>` exist or to **choose** a versioned PMCID, use `versions` first (examples and jq patterns live in `references/cli-versions.md`, linked from the main SKILL), not `metadata`. `metadata` does include a `version` number and versioned URLs for the resolved record, but it is not the right command for enumerating or picking versions.
+
+Example:
+```bash
+uvx pmc-toolkit metadata PMCxxxx.N
+```
+
+Example output:
+
+```json
+{
+  "pmcid": "PMCxxxx",
+  "version": N,
+  "pmid": 12345678,
+  "doi": "10.1234/example.doi",
+  "mid": null,
+  "title": "Example article title",
+  "citation": "Journal Name",
+  "is_pmc_openaccess": true/false,
+  "is_manuscript": true/false,
+  "is_historical_ocr": true/false,
+  "is_retracted": true/false,
+  "license_code": "license code",
+  "xml_url": "s3://pmc-oa-opendata/PMCxxxx.N/PMCxxxx.N.xml?md5=<hex>",
+  "pdf_url": "s3://pmc-oa-opendata/PMCxxxx.N/PMCxxxx.N.pdf?md5=<hex>",
+  "media_urls": [
+    "s3://pmc-oa-opendata/PMCxxxx.N/media-1.jpg?md5=<hex>",
+    ...
+  ],
+  "text_url": "s3://pmc-oa-opendata/PMCxxxx.N/PMCxxxx.N.txt?md5=<hex>"
+}
+```

pmc_toolkit-0.3.0/skills/pmc-toolkit/references/cli-parse-figures.md

@@ -0,0 +1,3 @@
+# parse: figures (`F*`)
+
+TODO

pmc_toolkit-0.3.0/skills/pmc-toolkit/references/cli-parse-references.md

@@ -0,0 +1,3 @@
+# parse: references (`R*`)
+
+TODO

pmc_toolkit-0.3.0/skills/pmc-toolkit/references/cli-parse-tables.md

@@ -0,0 +1,3 @@
+# parse: tables (`T*`)
+
+TODO

pmc_toolkit-0.3.0/skills/pmc-toolkit/references/cli-parse.md

@@ -0,0 +1,122 @@
+# CLI: `parse`
+
+Use `parse <PMCID.N>` after cached full-text XML exists. Run `fetch <PMCID.N> --ext xml` first when `<cache>/<PMCID.N>/<PMCID.N>.xml` is missing. The first run parses XML once, writes `<cache-root>/<PMCID.N>/.pmc-extracted-article.json`, and prints the extracted JSON; later runs reuse that cache unless `--force` is passed.
+
+Add `--cache-dir` or `PMC_TOOLKIT_CACHE` when the XML was fetched outside the default OS user cache. Keep the same cache root across `fetch` and `parse`.
+
+```bash
+uvx pmc-toolkit fetch PMCxxxx.N --ext xml
+uvx pmc-toolkit parse PMCxxxx.N
+```
+
+## Extracted JSON top-level keys
+
+- **article_info** — `journal`, `article_ids`, `title`, `publication_date`, `article_type`, `license`, `keywords`, `authors[]`, `abstract`, `funding_grants[]`
+- **content** — `paragraphs[]` and `sections[]`; items include `source_id`, `section_id`, `title`, `text`, `reference_ids`, `figure_ids`, and `table_ids`
+- **references** — `references[]` with `source_id`, `label`, `text`, `publication_type`, `identifiers`, `article_title`, `source`, `year`, `volume`, `issue`, and `pages`
+- **figures** — `figures[]` with `source_id`, `label`, `caption`, and `graphics`
+- **tables** — `tables[]` with `source_id`, `label`, `caption`, `rows`, and `footnotes`
+- **supporting_info** — `acknowledgements`, `competing_interests`, `data_availability`, `supplementary_media`, `author_notes`, `related_articles`, and `custom_metadata`
+
+## Narrow retrieval with jq
+
+**Start here.** Full `parse` output is large. Pipe it through jq and load only the slice you need.
+
+### Content outline (default first step)
+
+`scripts/content-outline.jq` returns a nested section tree: article title plus `section_id` and `title` for each section.
+
+```bash
+uvx pmc-toolkit parse PMCxxxx.N | jq -f scripts/content-outline.jq
+```
+
+Example output:
+
+```json
+{
+  "title": "journal title",
+  "sections": [
+    {
+      "section_id": "S1",
+      "title": "section title"
+    },
+    {
+      "section_id": "S2",
+      "title": "section title",
+      "sections": [
+        {
+          "section_id": "S3",
+          "title": "sub-section title"
+        }
+      ]
+    }
+  ]
+}
+```
+Use this to pick relevant sections (based on their titles) before loading detailed information. 
+The `section_id` values are XML source IDs (`S1`, `S2`, …) — use them with `scripts/query-id.jq` to fetch detailed section data.
+
+### Drill down by ID
+
+`scripts/query-id.jq` returns the first object whose `source_id` matches. After the content outline, pass a chosen ID:
+
+| Prefix | Meaning | Example |
+| --- | --- | --- |
+| `S*` | Section | `S3` |
+| `P*` | Paragraph | `P9` |
+| `F*` | Figure | `F1` |
+| `R*` | Reference | `R1` |
+| `T*` | Table | `T1` |
+
+**Section** — paragraph text and xref links for that section:
+
+```bash
+uvx pmc-toolkit parse PMCxxxx.N | jq --arg id "S3" -f scripts/query-id.jq
+```
+
+Example output:
+
+```json
+{
+  "source_id": "S3",
+  "section_id": "2.1",
+  "title": "sub-section title",
+  "paragraphs": [
+    {
+      "source_id": "P9",
+      "text": "paragraph text",
+      "reference_ids": ["R1", "R18"],
+      "figure_ids": ["F1", "F5"],
+      "table_ids": ["T1"]
+    }
+  ],
+  "sections": []
+}
+```
+
+Some sections are containers only. In the outline, `S2` (Results) has child sections but no paragraphs of its own — the text lives in `S3`, `S4`, etc. 
+Query those leaf `S*` IDs (sections with no nested `sections` in the outline), not the parent, to load only the subsection you need.
+
+**Figure, table, or reference** — same script, different ID prefix:
+
+```bash
+uvx pmc-toolkit parse PMCxxxx.N | jq --arg id "F1" -f scripts/query-id.jq
+uvx pmc-toolkit parse PMCxxxx.N | jq --arg id "R1" -f scripts/query-id.jq
+uvx pmc-toolkit parse PMCxxxx.N | jq --arg id "T1" -f scripts/query-id.jq
+```
+
+Use paragraph `reference_ids`, `figure_ids`, and `table_ids` to fetch linked entries with `scripts/query-id.jq`. Output shapes:
+
+- [cli-parse-references.md](cli-parse-references.md) — `R*` lookup
+- [cli-parse-figures.md](cli-parse-figures.md) — `F*` lookup
+- [cli-parse-tables.md](cli-parse-tables.md) — `T*` lookup
+
+### Reverse lookup by xref
+
+`query-id.jq` resolves an ID to its object. `reverse-lookup-xref.jq` finds every paragraph that cites a given reference, figure, or table. Pass `--arg xref` as `references`, `figures`, or `tables`:
+
+```bash
+uvx pmc-toolkit parse PMCxxxx.N | jq --arg xref references --arg id "R1" -f scripts/reverse-lookup-xref.jq
+uvx pmc-toolkit parse PMCxxxx.N | jq --arg xref figures --arg id "F1" -f scripts/reverse-lookup-xref.jq
+uvx pmc-toolkit parse PMCxxxx.N | jq --arg xref tables --arg id "T1" -f scripts/reverse-lookup-xref.jq
+```

pmc_toolkit-0.3.0/skills/pmc-toolkit/references/cli-versions.md

@@ -0,0 +1,33 @@
+# CLI: `versions`
+
+Use `versions <PMCID>` to list every published versioned PMCID string (`PMCxxxx.1`, `PMCxxxx.2`, …) for a **base** PMCID only. `versions` rejects versioned IDs.
+
+```bash
+uv run pmc-toolkit versions PMCxxxx
+```
+
+Example output shape:
+
+```json
+{
+  "pmcid": "PMCxxxx",
+  "versions": [
+    "PMCxxxx.1",
+    "PMCxxxx.2"
+  ]
+}
+```
+
+## Pick the latest `<PMCID.N>`
+
+```bash
+uv run pmc-toolkit versions PMCxxxx | jq -r '.versions[-1]'
+```
+
+## Pick a non-latest version
+
+Select an element of `.versions` by index (for example `.versions[0]` for the first published version).
+
+## Next steps
+
+After you have `<PMCID.N>`, continue with `metadata`, `files`, `fetch`, and `parse` as described in the main skill.

pmc_toolkit-0.3.0/skills/pmc-toolkit/scripts/content-outline.jq

@@ -0,0 +1,14 @@
+def drop_empty($o):
+  $o | with_entries(select(.value | if type == "array" then length > 0 else . != null end));
+
+def section:
+  drop_empty({
+    section_id: .source_id,
+    title: .title,
+    sections: [.sections[]? | section]
+  });
+
+{
+  title: .article_info.title,
+  sections: [.content.sections[]? | section]
+}

pmc_toolkit-0.3.0/skills/pmc-toolkit/scripts/query-id.jq

@@ -0,0 +1,3 @@
+..
+| objects
+| select(.source_id? == $id)

pmc_toolkit-0.3.0/skills/pmc-toolkit/scripts/reverse-lookup-xref.jq

@@ -0,0 +1,13 @@
+{
+  references: "reference_ids",
+  figures: "figure_ids",
+  tables: "table_ids"
+}[$xref] as $field
+| if $field == null then
+    error("xref must be references, figures, or tables")
+  else
+    [ .. | objects
+      | select(.[$field]? | index($id))
+      | {source_id, text, ($field): .[$field] }
+    ]
+  end

{pmc_toolkit-0.2.0 → pmc_toolkit-0.3.0}/src/pmc_toolkit/cli.py

@@ -140,8 +140,8 @@ def fetch(
     _emit_json(result.model_dump(mode="json"))
 
 
-@app.command("convert-xml")
-def convert_xml(
+@app.command("parse")
+def parse(
     requested_pmcid: str = typer.Argument(
         ...,
         help="PMC accession ID or version ID, e.g. PMC11370360 or PMC11370360.1",
@@ -158,22 +158,10 @@ def convert_xml(
         "-f",
         help="Recreate the extracted JSON cache from the cached XML.",
     ),
-    list_keys: bool = typer.Option(
-        False,
-        "--list-keys",
-        help="Print available extracted JSON keys and descriptions, then exit.",
-    ),
 ) -> None:
     """
-    Convert cached PMC full-text XML into cached extracted JSON.
+    Parse cached PMC full-text XML into cached extracted JSON.
     """
-    if list_keys:
-        from pmc_toolkit.xml_parse_utils import EXTRACT_OUTPUT_KEY_DESCRIPTIONS
-
-        typer.echo("Available extracted JSON keys:")
-        for key, description in EXTRACT_OUTPUT_KEY_DESCRIPTIONS.items():
-            typer.echo(f"- {key}: {description}")
-        return
 
     def build_result():
         from pmc_toolkit.xml_parse_api import ensure_extracted_article

{pmc_toolkit-0.2.0 → pmc_toolkit-0.3.0}/src/pmc_toolkit/xml_parse_utils.py

@@ -13,26 +13,6 @@ XMLParser = etree.XMLParser(
     remove_blank_text=True,
 )
 REFERENCE_SEPARATOR_PATTERN = re.compile(r"^[\s,;]+$")
-EXTRACT_OUTPUT_KEY_DESCRIPTIONS = {
-    "article_info": (
-        "article_info.journal, article_ids, title, publication_date, article_type, "
-        "license, keywords, authors[], abstract, and funding_grants[]"
-    ),
-    "content": (
-        "content.paragraphs[] and content.sections[]; objects include source_id, "
-        "section_id, title, text, reference_ids, figure_ids, and table_ids"
-    ),
-    "references": (
-        "references[] items with source_id, label, text, publication_type, "
-        "identifiers, article_title, source, year, volume, issue, and pages"
-    ),
-    "figures": "figures[] items with source_id, label, caption, and graphics",
-    "tables": "tables[] items with source_id, label, caption, rows, and footnotes",
-    "supporting_info": (
-        "acknowledgements, competing_interests, data_availability, "
-        "supplementary_media, author_notes, related_articles, and custom_metadata"
-    ),
-}
 
 
 def load_xml(path: Path) -> Any:

{pmc_toolkit-0.2.0 → pmc_toolkit-0.3.0}/uv.lock

@@ -251,7 +251,7 @@ wheels = [
 
 [[package]]
 name = "pmc-toolkit"
-version = "0.2.0"
+version = "0.3.0"
 source = { editable = "." }
 dependencies = [
     { name = "boto3" },