pmc-toolkit 0.1.0__py3-none-any.whl

pmc_toolkit/cache.py

@@ -0,0 +1,144 @@
+"""Local filesystem cache helpers for PMC metadata, manifests, and downloads."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pmc_toolkit.models import PMCMetadata
+
+OBJECT_KEYS_CACHE_FILENAME = ".pmc-object-keys.json"
+EXTRACTED_ARTICLE_CACHE_FILENAME = ".pmc-extracted-article.json"
+
+
+def default_cache_root() -> Path:
+    from platformdirs import user_cache_dir
+
+    return Path(user_cache_dir("pmc-toolkit", appauthor=False))
+
+
+def resolve_cache_root(cache_dir: Path | None = None) -> Path:
+    return Path(cache_dir) if cache_dir is not None else default_cache_root()
+
+
+def article_cache_dir(cache_root: Path, versioned_pmcid: str) -> Path:
+    return cache_root / versioned_pmcid
+
+
+def metadata_cache_path(cache_root: Path, versioned_pmcid: str) -> Path:
+    return article_cache_dir(cache_root, versioned_pmcid) / f"{versioned_pmcid}.json"
+
+
+def object_keys_cache_path(cache_root: Path, versioned_pmcid: str) -> Path:
+    return article_cache_dir(cache_root, versioned_pmcid) / OBJECT_KEYS_CACHE_FILENAME
+
+
+def extracted_article_cache_path(cache_root: Path, versioned_pmcid: str) -> Path:
+    return (
+        article_cache_dir(cache_root, versioned_pmcid)
+        / EXTRACTED_ARTICLE_CACHE_FILENAME
+    )
+
+
+def _read_json_file(path: Path) -> Any | None:
+    if not path.exists():
+        return None
+
+    import json
+
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def read_cached_metadata(cache_root: Path, versioned_pmcid: str) -> PMCMetadata | None:
+    payload = _read_json_file(metadata_cache_path(cache_root, versioned_pmcid))
+    if payload is None:
+        return None
+
+    from pmc_toolkit.models import PMCMetadata
+
+    return PMCMetadata.model_validate(payload)
+
+
+def write_cached_metadata(
+    cache_root: Path, versioned_pmcid: str, metadata: PMCMetadata
+) -> None:
+    path = metadata_cache_path(cache_root, versioned_pmcid)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(metadata.model_dump_json(indent=2), encoding="utf-8")
+
+
+def read_cached_object_keys(cache_root: Path, versioned_pmcid: str) -> list[str] | None:
+    payload = _read_json_file(object_keys_cache_path(cache_root, versioned_pmcid))
+    if payload is None:
+        return None
+    if not isinstance(payload, list) or not all(
+        isinstance(item, str) for item in payload
+    ):
+        raise ValueError(f"Invalid cached file listing for article: {versioned_pmcid}.")
+
+    return sorted(payload)
+
+
+def write_cached_object_keys(
+    cache_root: Path, versioned_pmcid: str, keys: list[str]
+) -> None:
+    import json
+
+    path = object_keys_cache_path(cache_root, versioned_pmcid)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(sorted(keys), indent=2), encoding="utf-8")
+
+
+def read_cached_extracted_article(
+    cache_root: Path, versioned_pmcid: str
+) -> dict[str, Any] | None:
+    path = extracted_article_cache_path(cache_root, versioned_pmcid)
+    try:
+        payload = _read_json_file(path)
+    except ValueError:
+        return None
+    if payload is None:
+        return None
+    if not isinstance(payload, dict):
+        return None
+    return payload
+
+
+def write_cached_extracted_article(
+    cache_root: Path, versioned_pmcid: str, data: dict[str, Any]
+) -> None:
+    import json
+
+    path = extracted_article_cache_path(cache_root, versioned_pmcid)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+
+
+def local_object_path(cache_root: Path, versioned_pmcid: str, key: str) -> Path:
+    """Return the local cache path for an S3 object key.
+
+    S3 object keys are remote-controlled opaque strings, not trusted filesystem paths.
+    A key may still start with the expected article prefix while using ``..`` or an
+    absolute path segment to escape the article cache directory, so this helper
+    enforces that the resolved destination remains inside that directory.
+    """
+    prefix = f"{versioned_pmcid}/"
+    if not key.startswith(prefix):
+        raise ValueError(
+            f"Object key {key!r} does not belong to article: {versioned_pmcid}."
+        )
+
+    relpath = key.removeprefix(prefix)
+    article_dir = article_cache_dir(cache_root, versioned_pmcid)
+
+    # Keep downloads contained to the article cache directory.
+    if Path(relpath).is_absolute():
+        raise ValueError(f"Unsafe object key path for article: {versioned_pmcid}.")
+
+    dest_path = (article_dir / relpath).resolve()
+    article_dir_resolved = article_dir.resolve()
+    if not dest_path.is_relative_to(article_dir_resolved):
+        raise ValueError(f"Unsafe object key path for article: {versioned_pmcid}.")
+
+    return dest_path

pmc_toolkit/cli.py

@@ -0,0 +1,196 @@
+from collections.abc import Callable
+import json
+from pathlib import Path
+from typing import Any, TypeVar
+
+import typer
+
+CommandResult = TypeVar("CommandResult")
+
+app = typer.Typer(
+    help="CLI for interacting with the PMC Open Data S3 bucket.",
+    no_args_is_help=True,
+)
+
+
+def _run_command(action: Callable[[], CommandResult]) -> CommandResult:
+    try:
+        return action()
+    except ValueError as exc:
+        typer.echo(f"Error: {exc}", err=True)
+        raise typer.Exit(code=2) from exc
+    except Exception as exc:
+        typer.echo(f"Error: {exc}", err=True)
+        raise typer.Exit(code=1) from exc
+
+
+def _emit_json(payload: Any) -> None:
+    typer.echo(json.dumps(payload, indent=2, ensure_ascii=False))
+
+
+@app.command("versions")
+def versions(
+    pmcid: str = typer.Argument(..., help="PMC accession ID, e.g. PMC11370360"),
+) -> None:
+    """
+    List all versions belonging to a PMCID.
+    """
+
+    def build_result():
+        from pmc_toolkit.storage_api import list_versions
+        from pmc_toolkit.validators import parse_pmcid
+
+        normalized_pmcid, version = parse_pmcid(pmcid)
+        if version is not None:
+            raise ValueError(
+                "The versions command expects a base PMCID like 'PMC11370360', not a versioned ID."
+            )
+        return list_versions(normalized_pmcid)
+
+    result = _run_command(build_result)
+    _emit_json(result.model_dump(mode="json"))
+
+
+@app.command("metadata")
+def metadata(
+    requested_pmcid: str = typer.Argument(
+        ...,
+        help="PMC accession ID or version ID, e.g. PMC11370360 or PMC11370360.1",
+    ),
+) -> None:
+    """
+    Fetch metadata for a PMC article identifier.
+    """
+
+    def build_result():
+        from pmc_toolkit.storage_api import get_metadata
+
+        return get_metadata(requested_pmcid)
+
+    result = _run_command(build_result)
+    _emit_json(result.model_dump(mode="json"))
+
+
+@app.command("files")
+def files(
+    requested_pmcid: str = typer.Argument(
+        ...,
+        help="PMC accession ID or version ID, e.g. PMC11370360 or PMC11370360.1",
+    ),
+) -> None:
+    """
+    List every object stored under a PMC article version's S3 prefix.
+    """
+
+    def build_result():
+        from pmc_toolkit.storage_api import list_files
+
+        return list_files(requested_pmcid)
+
+    result = _run_command(build_result)
+    _emit_json(result.model_dump(mode="json"))
+
+
+@app.command("fetch")
+def fetch(
+    requested_pmcid: str = typer.Argument(
+        ...,
+        help="PMC accession ID or version ID, e.g. PMC11370360 or PMC11370360.1",
+    ),
+    extensions: list[str] = typer.Option(
+        None,
+        "--ext",
+        "-e",
+        help=(
+            "Restrict download to these file extensions. Repeat the option or pass a "
+            "comma-separated list, e.g. -e pdf -e xml or -e pdf,xml."
+        ),
+    ),
+    cache_dir: Path = typer.Option(
+        None,
+        "--cache-dir",
+        envvar="PMC_TOOLKIT_CACHE",
+        help=(
+            "Cache root (default: OS user cache dir for pmc-toolkit, e.g. XDG on Linux, "
+            "Library/Caches on macOS, Local AppData on Windows). Files under <cache>/<PMCid.N>/."
+        ),
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        "-f",
+        help="Re-download files even when they already exist in the cache.",
+    ),
+) -> None:
+    """
+    Download all (or filtered) files for a PMC article version into a local cache.
+    """
+
+    def build_result():
+        from pmc_toolkit.storage_api import fetch_files
+
+        return fetch_files(
+            requested_pmcid,
+            cache_dir=cache_dir,
+            extensions=extensions,
+            force=force,
+        )
+
+    result = _run_command(build_result)
+    _emit_json(result.model_dump(mode="json"))
+
+
+@app.command("convert-xml")
+def convert_xml(
+    requested_pmcid: str = typer.Argument(
+        ...,
+        help="PMC accession ID or version ID, e.g. PMC11370360 or PMC11370360.1",
+    ),
+    cache_dir: Path = typer.Option(
+        None,
+        "--cache-dir",
+        envvar="PMC_TOOLKIT_CACHE",
+        help="Cache root containing <PMCid.N>/<PMCid.N>.xml.",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        "-f",
+        help="Recreate the extracted JSON cache from the cached XML.",
+    ),
+    list_keys: bool = typer.Option(
+        False,
+        "--list-keys",
+        help="Print available extracted JSON keys and descriptions, then exit.",
+    ),
+) -> None:
+    """
+    Convert cached PMC full-text XML into cached extracted JSON.
+    """
+    if list_keys:
+        from pmc_toolkit.xml_parse_utils import EXTRACT_OUTPUT_KEY_DESCRIPTIONS
+
+        typer.echo("Available extracted JSON keys:")
+        for key, description in EXTRACT_OUTPUT_KEY_DESCRIPTIONS.items():
+            typer.echo(f"- {key}: {description}")
+        return
+
+    def build_result():
+        from pmc_toolkit.xml_parse_api import ensure_extracted_article
+
+        return ensure_extracted_article(
+            requested_pmcid,
+            cache_dir=cache_dir,
+            force=force,
+        )
+
+    result = _run_command(build_result)
+    _emit_json(result.data)
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()

pmc_toolkit/models.py

@@ -0,0 +1,56 @@
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class PMCVersions(BaseModel):
+    pmcid: str
+    versions: list[str]
+
+
+class PMCMetadata(BaseModel):
+    pmcid: str
+    version: int
+    pmid: int | None = None
+    doi: str | None = None
+    mid: str | None = None
+    title: str | None = None
+    citation: str | None = None
+    is_pmc_openaccess: bool
+    is_manuscript: bool
+    is_historical_ocr: bool
+    is_retracted: bool
+    license_code: str | None = None
+    xml_url: str
+    pdf_url: str | None = None
+    media_urls: list[str] = Field(default_factory=list)
+    text_url: str
+
+
+class PMCFiles(BaseModel):
+    versioned_pmcid: str
+    keys: list[str]
+
+
+class FetchAction(str, Enum):
+    DOWNLOADED = "downloaded"
+    SKIPPED = "skipped"
+
+
+class PMCFetchFile(BaseModel):
+    key: str
+    local_path: str
+    action: FetchAction
+
+
+class PMCFetchResult(BaseModel):
+    versioned_pmcid: str
+    cache_dir: str
+    files: list[PMCFetchFile]
+
+
+class PMCExtractResult(BaseModel):
+    versioned_pmcid: str
+    xml_path: str
+    data: dict[str, Any]

pmc_toolkit/storage_api.py

@@ -0,0 +1,108 @@
+"""Public API for the PMC open-access **S3 dataset** and **local download cache**.
+
+CLI commands use this API; low-level S3 helpers live in :mod:`pmc_toolkit.storage_utils`
+and local cache helpers live in :mod:`pmc_toolkit.cache`."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pmc_toolkit import cache as storage_cache
+from pmc_toolkit import storage_utils
+from pmc_toolkit.validators import parse_pmcid
+
+if TYPE_CHECKING:
+    from pmc_toolkit.models import (
+        PMCFetchFile,
+        PMCFetchResult,
+        PMCFiles,
+        PMCMetadata,
+        PMCVersions,
+    )
+
+
+def list_versions(pmcid: str) -> PMCVersions:
+    from pmc_toolkit.models import PMCVersions
+
+    base_pmcid, version = parse_pmcid(pmcid)
+    if version is not None:
+        raise ValueError(
+            "list_versions expects a base PMCID like 'PMC11370360', not a versioned ID."
+        )
+    versions = storage_utils.list_versioned_pmcids(base_pmcid)
+    return PMCVersions(
+        pmcid=base_pmcid,
+        versions=sorted(set(versions), key=storage_utils.version_number),
+    )
+
+
+def get_metadata(requested_pmcid: str) -> PMCMetadata:
+    cache_root = storage_cache.resolve_cache_root()
+    versioned_pmcid = storage_utils.resolve_versioned_pmcid(requested_pmcid)
+    cached = storage_cache.read_cached_metadata(cache_root, versioned_pmcid)
+
+    if cached is not None:
+        return cached
+
+    metadata = storage_utils.read_metadata(versioned_pmcid)
+    storage_cache.write_cached_metadata(cache_root, versioned_pmcid, metadata)
+    return metadata
+
+
+def list_files(requested_pmcid: str) -> PMCFiles:
+    from pmc_toolkit.models import PMCFiles
+
+    cache_root = storage_cache.resolve_cache_root()
+    versioned_pmcid = storage_utils.resolve_versioned_pmcid(requested_pmcid)
+    keys = storage_utils.read_or_cache_object_keys(cache_root, versioned_pmcid)
+
+    return PMCFiles(versioned_pmcid=versioned_pmcid, keys=keys)
+
+
+def fetch_files(
+    requested_pmcid: str,
+    cache_dir: Path | None = None,
+    extensions: list[str] | None = None,
+    force: bool = False,
+) -> PMCFetchResult:
+    from pmc_toolkit.models import FetchAction, PMCFetchFile, PMCFetchResult
+
+    cache_root = storage_cache.resolve_cache_root(cache_dir)
+    versioned_pmcid = storage_utils.resolve_versioned_pmcid(requested_pmcid)
+    all_keys = storage_utils.read_or_cache_object_keys(cache_root, versioned_pmcid)
+
+    normalized = storage_utils.normalize_extensions(extensions)
+    keys = [
+        key for key in all_keys if storage_utils.key_matches_extensions(key, normalized)
+    ]
+
+    article_dir = storage_cache.article_cache_dir(cache_root, versioned_pmcid)
+    article_dir.mkdir(parents=True, exist_ok=True)
+
+    results: list[PMCFetchFile] = []
+    mkdir_done: set[Path] = set()
+
+    for key in keys:
+        dest = storage_cache.local_object_path(cache_root, versioned_pmcid, key)
+        parent = dest.parent
+        if parent not in mkdir_done:
+            parent.mkdir(parents=True, exist_ok=True)
+            mkdir_done.add(parent)
+
+        if dest.exists() and not force:
+            results.append(
+                PMCFetchFile(key=key, local_path=str(dest), action=FetchAction.SKIPPED)
+            )
+            continue
+
+        storage_utils.download_object(key, dest)
+        results.append(
+            PMCFetchFile(key=key, local_path=str(dest), action=FetchAction.DOWNLOADED)
+        )
+
+    return PMCFetchResult(
+        versioned_pmcid=versioned_pmcid,
+        cache_dir=str(article_dir),
+        files=results,
+    )

pmc_toolkit/storage_utils.py

@@ -0,0 +1,169 @@
+"""Internal S3 helpers for ``storage_api``; local cache helpers live in
+:mod:`pmc_toolkit.cache`. Not the Python import surface—use
+:mod:`pmc_toolkit.storage_api`."""
+
+from __future__ import annotations
+
+from functools import cache
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pmc_toolkit import cache as storage_cache
+from pmc_toolkit.validators import parse_pmcid
+
+if TYPE_CHECKING:
+    from pmc_toolkit.models import PMCMetadata
+    from types_boto3_s3.client import S3Client
+
+BUCKET = "pmc-oa-opendata"
+REGION = "us-east-1"
+
+
+# Shared S3 setup
+@cache
+def _get_s3_client() -> "S3Client":
+    import boto3
+    from botocore import UNSIGNED
+    from botocore.config import Config
+
+    return boto3.client(
+        "s3", region_name=REGION, config=Config(signature_version=UNSIGNED)
+    )
+
+
+# Version resolution helpers
+def version_number(versioned_pmcid: str) -> int:
+    return int(versioned_pmcid.rsplit(".", 1)[1])
+
+
+def list_versioned_pmcids(pmcid: str) -> list[str]:
+    prefix = f"{pmcid}."
+
+    s3 = _get_s3_client()
+    paginator = s3.get_paginator("list_objects_v2")
+    pages = paginator.paginate(
+        Bucket=BUCKET,
+        Prefix=prefix,
+        Delimiter="/",
+    )
+
+    versions: list[str] = []
+
+    for page in pages:
+        for item in page.get("CommonPrefixes", []):
+            raw_prefix = item["Prefix"]
+            version = raw_prefix.rstrip("/")
+            if version:
+                versions.append(version)
+
+    return versions
+
+
+def _latest_versioned_pmcid(pmcid: str) -> str:
+    """Return the highest available article version published for a base PMCID."""
+    versions = list_versioned_pmcids(pmcid)
+
+    if not versions:
+        raise ValueError(f"No versions found for PMCID: {pmcid}.")
+
+    return max(versions, key=version_number)
+
+
+def resolve_versioned_pmcid(requested_pmcid: str) -> str:
+    """Resolve a PMCID input to an explicit version, using the latest version when omitted."""
+    pmcid, version = parse_pmcid(requested_pmcid)
+    return (
+        f"{pmcid}.{version}" if version is not None else _latest_versioned_pmcid(pmcid)
+    )
+
+
+# Metadata S3 helpers
+def read_metadata(versioned_pmcid: str) -> PMCMetadata:
+    """Fetch article metadata from the S3 metadata index for a specific version."""
+    key = f"metadata/{versioned_pmcid}.json"
+
+    import json
+
+    from botocore.exceptions import ClientError
+    from pmc_toolkit.models import PMCMetadata
+
+    s3 = _get_s3_client()
+
+    try:
+        response = s3.get_object(Bucket=BUCKET, Key=key)
+    except ClientError as exc:
+        error_code = exc.response.get("Error", {}).get("Code")
+        if error_code in {"NoSuchKey", "404"}:
+            raise ValueError(
+                f"No metadata found for article: {versioned_pmcid}."
+            ) from exc
+        raise
+
+    payload = json.loads(response["Body"].read())
+    return PMCMetadata.model_validate(payload)
+
+
+# Object-key listing helpers
+def list_object_keys(versioned_pmcid: str) -> list[str]:
+    prefix = f"{versioned_pmcid}/"
+
+    s3 = _get_s3_client()
+    paginator = s3.get_paginator("list_objects_v2")
+    pages = paginator.paginate(Bucket=BUCKET, Prefix=prefix)
+
+    keys: list[str] = []
+    for page in pages:
+        for item in page.get("Contents", []):
+            key = item.get("Key")
+            if key and not key.endswith("/"):
+                keys.append(key)
+
+    return sorted(keys)
+
+
+def read_or_cache_object_keys(cache_root: Path, versioned_pmcid: str) -> list[str]:
+    """Return cached object keys when available, otherwise list S3 and persist the result."""
+    keys = storage_cache.read_cached_object_keys(cache_root, versioned_pmcid)
+    if keys is not None:
+        return keys
+
+    keys = list_object_keys(versioned_pmcid)
+    if not keys:
+        raise ValueError(f"No files found for article: {versioned_pmcid}.")
+
+    storage_cache.write_cached_object_keys(cache_root, versioned_pmcid, keys)
+    return keys
+
+
+# Fetch filtering and downloads
+def normalize_extensions(extensions: list[str] | None) -> set[str] | None:
+    """Normalize repeated or comma-separated extension filters into a lowercase suffix set."""
+    if not extensions:
+        return None
+    normalized = {
+        part.strip().lower().lstrip(".")
+        for ext in extensions
+        for part in ext.split(",")
+        if part.strip()
+    }
+    return normalized or None
+
+
+def key_matches_extensions(key: str, extensions: set[str] | None) -> bool:
+    if extensions is None:
+        return True
+    filename = key.rsplit("/", 1)[-1]
+    _, _, ext = filename.rpartition(".")
+    return bool(ext) and ext.lower() in extensions
+
+
+def download_object(key: str, dest: Path) -> None:
+    from botocore.exceptions import ClientError
+
+    try:
+        _get_s3_client().download_file(BUCKET, key, str(dest))
+    except ClientError as exc:
+        error_code = exc.response.get("Error", {}).get("Code")
+        if error_code in {"NoSuchKey", "404"}:
+            raise ValueError(f"No object found for key: {key!r}.") from exc
+        raise