pharmadata 0.1.0__py3-none-any.whl

pharmadata/__init__.py

@@ -0,0 +1,41 @@
+"""Access CDISC ADaM and SDTM test datasets in Python for clinical programming."""
+
+import importlib
+import importlib.metadata as _metadata
+
+from pharmadata._core.data import COLLECTIONS as _COLLECTIONS
+from pharmadata._core.meta import DatasetMeta
+from pharmadata._core.output_format import (
+    get_output_format,
+    set_output_arrow,
+    set_output_pandas,
+    set_output_polars,
+)
+from pharmadata._core.output_format import (
+    use_output_format as output_format,
+)
+
+# The version lives in pyproject.toml
+try:
+    __version__ = _metadata.version("pharmadata")
+except (
+    _metadata.PackageNotFoundError
+):  # pragma: no cover - only in an unpacked source tree
+    __version__ = "0.0.0"
+
+# Import every discovered collection so ``pharmadata.<name>`` resolves. The set
+# comes from ``_data``, not a hardcoded list, so a new collection appears here
+# without an edit.
+for _name in _COLLECTIONS:
+    globals()[_name] = importlib.import_module(f"pharmadata.{_name}")
+
+__all__ = [
+    "DatasetMeta",
+    "__version__",
+    "get_output_format",
+    "output_format",
+    "set_output_arrow",
+    "set_output_pandas",
+    "set_output_polars",
+    *sorted(_COLLECTIONS),
+]

pharmadata/_core/__init__.py

@@ -0,0 +1 @@
+"""Core infrastructure: registry, metadata, output format, and collection wiring."""

pharmadata/_core/collection.py

@@ -0,0 +1,119 @@
+"""Wire a collection onto its module: lazy dataset attributes, ``__all__``, ``__dir__``."""
+
+from __future__ import annotations
+
+import sys
+from functools import cache
+from typing import TYPE_CHECKING, cast
+
+import polars as pl
+
+from pharmadata._core import data, meta, output_format
+
+if TYPE_CHECKING:
+    # pandas is an optional extra; pyarrow is only named by the annotations
+    # (both live behind ``data``, which reads and converts the arrow base).
+    import pandas as pd
+    import pyarrow as pa
+
+# The functions every collection module defines (and that __all__ lists).
+HELPER_FUNCTIONS = (
+    "list_datasets",
+    "meta",
+    "meta_tables",
+    "meta_columns",
+    "meta_specs",
+    "to_dict",
+)
+
+
+@cache
+def _load_polars(collection: str, name: str) -> pl.DataFrame:
+    """The process-wide cached polars frame; callers get a clone via ``load``."""
+    # from_arrow of a Table is a DataFrame; the stub returns the wider union.
+    return cast("pl.DataFrame", pl.from_arrow(data.load_dataset(collection, name)))
+
+
+@cache
+def _load_pandas(collection: str, name: str) -> pd.DataFrame:
+    """The process-wide cached pandas frame; callers get a copy via ``load``."""
+    # pyarrow's parquet reader initialises the pandas shim, which crashes when
+    # pandas is absent; ``load`` calls ``data.require_pandas`` before this runs.
+    return cast(
+        "pd.DataFrame", data.to_output(data.load_dataset(collection, name), "pandas")
+    )
+
+
+def load(collection: str, name: str) -> pa.Table | pl.DataFrame | pd.DataFrame:
+    """One dataset on the active output format.
+
+    The arrow base is immutable, so ``"arrow"`` shares the cached table; polars
+    returns a fresh clone and pandas a deep copy, so a mutation never poisons
+    the cache another caller reads.
+    """
+    fmt = output_format.get_output_format()
+    if fmt == "arrow":
+        return data.load_dataset(collection, name)
+    if fmt == "pandas":
+        # before load_dataset: pyarrow would initialise the pandas shim first
+        data.require_pandas()
+        frame: pa.Table | pl.DataFrame | pd.DataFrame = _load_pandas(
+            collection, name
+        ).copy()
+    else:
+        frame = _load_polars(collection, name).clone()
+    # a clone/copy does not carry the cached frame's instance docstring
+    frame.__doc__ = data.dataset_doc(collection, name)
+    return frame
+
+
+def load_all(
+    collection: str,
+) -> dict[str, pa.Table | pl.DataFrame | pd.DataFrame]:
+    """Every dataset of a collection keyed by name, honoring the active output format.
+
+    The values are polars frames by default, pyarrow Tables when
+    ``output_format="arrow"`` is active, and pandas frames when
+    ``output_format="pandas"`` is active.
+    """
+    return {name: load(collection, name) for name in data.list_datasets(collection)}
+
+
+def install(module_name: str, spec: data.Collection) -> None:
+    """Expose *spec*'s datasets as lazy attributes on the module *module_name*."""
+    module = sys.modules.get(module_name)
+    if module is None:  # pragma: no cover - only under an unusual loader
+        msg = f"{module_name} is not in sys.modules; cannot install collection API"
+        raise RuntimeError(msg)
+
+    names = data.dataset_names(spec.name)
+    sorted_names = data.list_datasets(spec.name)
+    module.__dict__["__all__"] = [
+        *HELPER_FUNCTIONS,
+        *sorted_names,
+        *(f"{name}_meta" for name in sorted_names),
+    ]
+
+    def __getattr__(
+        name: str,
+    ) -> pa.Table | pl.DataFrame | pd.DataFrame | meta.DatasetMeta:
+        """The dataset named *name*, or its metadata as ``<name>_meta``."""
+        if name.startswith("__") and name.endswith("__"):
+            # keep introspection (copy, pickle, IPython) on its normal path
+            raise AttributeError(name)
+        if name in names:
+            return load(spec.name, name)
+        stem = name.removesuffix("_meta")
+        if name.endswith("_meta") and stem in names:
+            return meta.build(spec.name, stem)
+        msg = (
+            f"module {module_name!r} has no dataset {name!r}; "
+            f"available: {', '.join(sorted_names)}"
+        )
+        raise AttributeError(msg)
+
+    def __dir__() -> list[str]:
+        return sorted(module.__dict__["__all__"])
+
+    module.__dict__["__getattr__"] = __getattr__
+    module.__dict__["__dir__"] = __dir__

pharmadata/_core/data.py

@@ -0,0 +1,295 @@
+"""Dataset registry and loading: which datasets exist, their metadata, and I/O."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from functools import cache
+from pathlib import Path
+from typing import TYPE_CHECKING, TypedDict, cast
+
+import polars as pl
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from pharmadata._core.output_format import OutputFormat, get_output_format
+
+if TYPE_CHECKING:
+    # pandas is an optional extra; only named by the annotations below.
+    import pandas as pd
+
+
+@dataclass(frozen=True, slots=True)
+class Collection:
+    """One named group of datasets: module, title, and where it comes from.
+
+    ``url`` and ``example`` are optional: a bare new collection needs only a
+    ``title`` and ``source``. ``example`` falls back to the first dataset.
+    """
+
+    name: str  # the submodule and the _data directory name
+    title: str  # for the generated docs and the nav
+    source: str  # the upstream source, named on its index page
+    url: str = ""  # where that source is published
+    example: str = ""  # a representative dataset, used in docstring examples
+
+
+# The _meta.json contract: one entry per dataset, one spec per column.
+class ColumnSpec(TypedDict):
+    """One column's entry in a collection's ``_meta.json``."""
+
+    label: str | None
+    type: str
+    mandatory: str | None
+    role: str | None
+
+
+class DatasetMetaEntry(TypedDict):
+    """One dataset's entry in a collection's ``_meta.json``."""
+
+    label: str | None
+    structure: str | None
+    n_rows: int
+    columns: dict[str, ColumnSpec]
+
+
+# The directory the shipped data lives in, a sibling of this ``_core`` package.
+_DATA_ROOT = Path(__file__).resolve().parent.parent / "_data"
+
+
+def _resolve_example(child: Path, example: str) -> str:
+    """Return *example*, else the first dataset declared in ``_meta.json``."""
+    if example:
+        return example
+    meta_path = child / "_meta.json"
+    if not meta_path.is_file():
+        return ""
+    datasets = json.loads(meta_path.read_text(encoding="utf-8"))
+    return next(iter(sorted(datasets)), "")
+
+
+def _discover_collections() -> dict[str, Collection]:
+    """Every collection declared by a ``_collection.json`` under ``_data``.
+
+    A directory becomes a collection by dropping a ``_collection.json`` beside
+    its ``_meta.json``. Only ``title`` and ``source`` are required; ``url`` and
+    ``example`` are optional, and an omitted ``example`` falls back to the first
+    dataset. Nothing is hardcoded, so a new dataset group is picked up without
+    editing Python.
+    """
+    found: dict[str, Collection] = {}
+    if not _DATA_ROOT.is_dir():  # pragma: no cover - the data ships with the package
+        return found
+    for child in sorted(_DATA_ROOT.iterdir()):
+        config = child / "_collection.json"
+        if child.is_dir() and config.is_file():
+            raw = json.loads(config.read_text(encoding="utf-8"))
+            found[child.name] = Collection(
+                name=child.name,
+                title=str(raw["title"]),
+                source=str(raw["source"]),
+                url=str(raw.get("url", "")),
+                example=_resolve_example(child, str(raw.get("example", ""))),
+            )
+    return found
+
+
+# The single source of truth for which collections exist: read from the data.
+COLLECTIONS: dict[str, Collection] = _discover_collections()
+
+
+@cache
+def data_dir(collection: str) -> Path:
+    """Directory holding the parquet files and metadata of a collection."""
+    if collection not in COLLECTIONS:
+        msg = (
+            f"unknown collection {collection!r}; expected one of {sorted(COLLECTIONS)}"
+        )
+        raise ValueError(msg)
+    return _DATA_ROOT / collection
+
+
+@cache
+def raw_meta(collection: str) -> dict[str, DatasetMetaEntry]:
+    """Raw metadata of a collection: dataset -> label/structure/columns."""
+    path = data_dir(collection) / "_meta.json"
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+@cache
+def dataset_names(collection: str) -> frozenset[str]:
+    """Names of the datasets available in a collection."""
+    return frozenset(raw_meta(collection))
+
+
+def check_dataset(collection: str, name: str) -> None:
+    """Raise a helpful error when *name* is not a dataset in *collection*."""
+    if name not in dataset_names(collection):
+        msg = (
+            f"no dataset {name!r} in collection {collection!r}; "
+            f"available: {', '.join(list_datasets(collection))}"
+        )
+        raise ValueError(msg)
+
+
+def _meta_entry(collection: str, name: str) -> DatasetMetaEntry:
+    """The raw metadata dict of one dataset, after validating it exists."""
+    check_dataset(collection, name)
+    return raw_meta(collection)[name]
+
+
+def list_datasets(collection: str) -> list[str]:
+    """Names of the datasets available in a collection, sorted alphabetically."""
+    return sorted(dataset_names(collection))
+
+
+@cache
+def load_dataset(collection: str, name: str) -> pa.Table:
+    """Load a dataset as an immutable pyarrow Table, the internal base.
+
+    pyarrow reads the parquet on every platform, including the browser, where
+    polars' own parquet reader is unavailable; the served frame is the arrow
+    table converted on demand by :func:`to_output`.
+    """
+    check_dataset(collection, name)
+    return pq.read_table(data_dir(collection) / f"{name}.parquet")
+
+
+# The one pandas-missing message, raised wherever "pandas" output is asked for.
+PANDAS_REQUIRED_MSG = (
+    'pandas is required for output_format="pandas"; '
+    'install with: pip install "pharmadata[pandas]"'
+)
+
+
+def require_pandas() -> None:
+    """Raise ImportError unless pandas can be imported.
+
+    ``sys.modules["pandas"] = None`` (the test fixture for a missing pandas)
+    makes ``import pandas`` return None without raising, so check the bound
+    name. Readers of parquet must call this first: pyarrow's reader
+    initialises the pandas shim and crashes on it when pandas is absent.
+    """
+    try:
+        import pandas as pd
+    except ImportError:
+        pd = None  # type: ignore[assignment]
+    if pd is None:
+        msg = PANDAS_REQUIRED_MSG
+        raise ImportError(msg)
+
+
+def to_output(
+    table: pa.Table, fmt: OutputFormat
+) -> pa.Table | pl.DataFrame | pd.DataFrame:
+    """Convert the arrow base into the requested output format.
+
+    ``"arrow"`` hands the table back untouched; polars crosses the Arrow C data
+    interface with ``pl.from_arrow``; pandas converts through ``to_pandas``,
+    naming the extra to install when pandas is absent.
+    """
+    if fmt == "arrow":
+        return table
+    if fmt == "pandas":
+        require_pandas()
+        return table.to_pandas()
+    if fmt == "polars":
+        # a Table converts to a DataFrame; from_arrow's stub returns the wider union.
+        return cast("pl.DataFrame", pl.from_arrow(table))
+    msg = f"unknown output format: {fmt!r}"  # pragma: no cover
+    raise ValueError(msg)  # pragma: no cover
+
+
+@cache
+def dataset_doc(collection: str, name: str) -> str:
+    """The docstring of a dataset, generated from its CDISC metadata."""
+    entry = _meta_entry(collection, name)
+    spec = COLLECTIONS[collection]
+    rows, cols = shape(collection, name)
+    lines = [
+        f"{name} - {entry['label'] or name}",
+        "",
+        f"{spec.title}, from {spec.source} (<{spec.url}>).",
+        f"{rows} rows x {cols} columns.",
+    ]
+    if entry["structure"]:
+        lines.append(f"Structure: {entry['structure']}.")
+    lines += ["", "Variables", "---------"]
+    lines += [
+        f"{column} ({info['type']}): {info['label'] or ''}".rstrip()
+        for column, info in entry["columns"].items()
+    ]
+    return "\n".join(lines) + "\n"
+
+
+@cache
+def shape(collection: str, name: str) -> tuple[int, int]:
+    """Row and column counts of a dataset, without loading any data."""
+    entry = _meta_entry(collection, name)
+    return entry["n_rows"], len(entry["columns"])
+
+
+@cache
+def _meta_tables_table(collection: str) -> pa.Table:
+    """Arrow base of the dataset-level metadata: one row per dataset."""
+    meta = raw_meta(collection)
+    names = list_datasets(collection)
+    return pa.table(
+        {
+            "dataset": names,
+            "label": [meta[n]["label"] for n in names],
+            "n_rows": [meta[n]["n_rows"] for n in names],
+            "n_cols": [len(meta[n]["columns"]) for n in names],
+        }
+    )
+
+
+@cache
+def _meta_specs_table(collection: str) -> pa.Table:
+    """Arrow base of the column-level specifications of every dataset."""
+    meta = raw_meta(collection)
+    dataset: list[str] = []
+    variable: list[str] = []
+    label: list[str | None] = []
+    column_type: list[str] = []
+    mandatory: list[str | None] = []
+    role: list[str | None] = []
+    for name in list_datasets(collection):
+        for col, info in meta[name]["columns"].items():
+            dataset.append(name)
+            variable.append(col)
+            label.append(info["label"])
+            column_type.append(info["type"])
+            mandatory.append(info.get("mandatory"))
+            role.append(info.get("role"))
+    return pa.table(
+        {
+            "dataset": dataset,
+            "variable": variable,
+            "label": label,
+            "type": column_type,
+            "mandatory": mandatory,
+            "role": role,
+        }
+    )
+
+
+def meta_tables(collection: str) -> pa.Table | pl.DataFrame | pd.DataFrame:
+    """Dataset-level metadata: one row per dataset, in the active format."""
+    return to_output(_meta_tables_table(collection), get_output_format())
+
+
+def meta_columns(collection: str) -> pa.Table | pl.DataFrame | pd.DataFrame:
+    """Column-level metadata of every dataset (long format), in the active format.
+
+    The specifications minus the ``mandatory`` and ``role`` columns.
+    """
+    table = _meta_specs_table(collection).select(
+        ["dataset", "variable", "label", "type"]
+    )
+    return to_output(table, get_output_format())
+
+
+def meta_specs(collection: str) -> pa.Table | pl.DataFrame | pd.DataFrame:
+    """Column-level specifications of every dataset (long format), active format."""
+    return to_output(_meta_specs_table(collection), get_output_format())

pharmadata/_core/meta.py

@@ -0,0 +1,159 @@
+"""Metadata of one dataset: the ``DatasetMeta`` value object."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import cache
+from typing import TYPE_CHECKING, ClassVar, Literal
+
+import pyarrow as pa
+
+from pharmadata._core import data, output_format
+
+if TYPE_CHECKING:
+    import pandas as pd
+    import polars as pl
+
+
+@dataclass(frozen=True, slots=True)
+class DatasetMeta:
+    """Metadata of one dataset, in the style of pyreadstat's metadata object.
+
+    A read-only view: the field names and parallel-tuple layout follow
+    pyreadstat, extended with the CDISC dimensions (``structure``,
+    ``mandatory``, ``role``). The data itself is reached through the module
+    attribute of the same name (e.g. ``pharmaverseadam.adsl``).
+
+    Attributes
+    ----------
+    dataset : str
+        The dataset name, e.g. ``"adsl"``.
+    collection : str
+        The collection it belongs to, e.g. ``"pharmaverseadam"``.
+    file_label : str or None
+        The dataset label, equivalent to a SAS dataset label.
+    structure : str or None
+        The CDISC structure, e.g. ``"One record per subject"``; None when the
+        source specifications do not provide one.
+    number_rows : int
+        Row count, read from the parquet footer without loading any data.
+    number_columns : int
+        Column count, i.e. ``len(column_names)``.
+    column_names : tuple of str
+        The column names, in column order.
+    column_labels : tuple of str or None
+        The column labels, parallel to ``column_names``.
+    column_types : tuple of str or None
+        The column types, parallel to ``column_names``.
+    column_mandatory : tuple of str or None
+        The CDISC mandatory flags, parallel to ``column_names``.
+    column_roles : tuple of str or None
+        The CDISC roles, parallel to ``column_names``.
+    file_format : str
+        Always ``"parquet"``, the storage format of every dataset.
+    file_encoding : str
+        Always ``"utf-8"``: the encoding of the shipped parquet data, not of
+        the original source file.
+
+    Examples
+    --------
+    >>> meta = pharmaverseadam.meta("adsl")
+    >>> meta.dataset
+    'adsl'
+    >>> meta.file_label
+    'Subject Level Analysis'
+    >>> meta.column_names_to_labels["STUDYID"]
+    'Study Identifier'
+    >>> (meta.number_rows, meta.number_columns)
+    (306, 55)
+    """
+
+    dataset: str
+    collection: str
+    file_label: str | None
+    structure: str | None
+    number_rows: int
+    column_names: tuple[str, ...]
+    column_labels: tuple[str | None, ...]
+    column_types: tuple[str | None, ...]
+    column_mandatory: tuple[str | None, ...]
+    column_roles: tuple[str | None, ...]
+
+    file_format: ClassVar[Literal["parquet"]] = "parquet"
+    file_encoding: ClassVar[Literal["utf-8"]] = "utf-8"
+
+    def __repr__(self) -> str:
+        """One line: which dataset, and how big."""
+        return (
+            f"DatasetMeta({self.collection}.{self.dataset}, "
+            f"{self.number_rows} rows x {self.number_columns} columns)"
+        )
+
+    @property
+    def number_columns(self) -> int:
+        """Column count, i.e. ``len(column_names)``."""
+        return len(self.column_names)
+
+    @property
+    def column_names_to_labels(self) -> dict[str, str | None]:
+        """The column names mapped to their labels, in column order.
+
+        Returns
+        -------
+        dict[str, str or None]
+            A dict mapping each column name to its CDISC label.
+
+        Examples
+        --------
+        >>> pharmaverseadam.adsl_meta.column_names_to_labels["AGE"]
+        'Age'
+        """
+        return dict(zip(self.column_names, self.column_labels, strict=True))
+
+    @property
+    def specs(self) -> pa.Table | pl.DataFrame | pd.DataFrame:
+        """Column-level specifications of the dataset, in the active format.
+
+        Returns
+        -------
+        pa.Table or pl.DataFrame or pd.DataFrame
+            One row per variable with the columns ``variable, label, type,
+            mandatory, role`` (mandatory/role are null when the source
+            specifications do not provide them). The frame is served in the
+            active output format, like the dataset attributes.
+
+        Examples
+        --------
+        >>> pharmaverseadam.adsl_meta.specs.columns
+        ['variable', 'label', 'type', 'mandatory', 'role']
+        """
+        table = pa.table(
+            {
+                "variable": self.column_names,
+                "label": self.column_labels,
+                "type": self.column_types,
+                "mandatory": self.column_mandatory,
+                "role": self.column_roles,
+            }
+        )
+        return data.to_output(table, output_format.get_output_format())
+
+
+@cache
+def build(collection: str, name: str) -> DatasetMeta:
+    """The DatasetMeta of one dataset, built once and shared process-wide."""
+    entry = data._meta_entry(collection, name)
+    n_rows, _ = data.shape(collection, name)
+    columns = entry["columns"]
+    return DatasetMeta(
+        dataset=name,
+        collection=collection,
+        file_label=entry.get("label"),
+        structure=entry.get("structure"),
+        number_rows=n_rows,
+        column_names=tuple(columns),
+        column_labels=tuple(info.get("label") for info in columns.values()),
+        column_types=tuple(info.get("type") for info in columns.values()),
+        column_mandatory=tuple(info.get("mandatory") for info in columns.values()),
+        column_roles=tuple(info.get("role") for info in columns.values()),
+    )