hca-anndata-tools 0.2.0__tar.gz

hca_anndata_tools-0.2.0/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.4
+Name: hca-anndata-tools
+Version: 0.2.0
+Summary: Library for inspection, summarization, and statistics of AnnData h5ad files
+License: MIT
+Author: Clever Canary
+Author-email: dave@clevercanary.com
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: anndata (>=0.10,<1)
+Requires-Dist: h5py (>=3.8)
+Requires-Dist: matplotlib (>=3.7)
+Requires-Dist: numpy (<3)
+Requires-Dist: pandas (>2,<3)
+Requires-Dist: pydantic (>=2,<3)
+Requires-Dist: scanpy (>=1.10)
+Project-URL: Homepage, https://github.com/clevercanary/hca-validation-tools
+Description-Content-Type: text/markdown
+
+# hca-anndata-tools
+
+Library for inspection, summarization, and statistics of AnnData h5ad files.
+
+## Installation
+
+```bash
+pip install hca-anndata-tools
+```
+
+## Usage
+
+```python
+from hca_anndata_tools import get_summary, get_descriptive_stats, locate_files
+
+# Find h5ad files
+files = locate_files("/path/to/data")
+
+# Get structural overview
+summary = get_summary("/path/to/my_atlas.h5ad")
+
+# Get column statistics
+stats = get_descriptive_stats("/path/to/my_atlas.h5ad", columns=["cell_type"], value_counts=True)
+```
+
+## API
+
+- **locate_files** - Find all .h5ad files in a directory
+- **get_summary** - Structural overview: cell/gene counts, columns, embeddings, layers
+- **get_storage_info** - HDF5 compression, chunking, sparse format details
+- **get_descriptive_stats** - Descriptive statistics and value counts for obs/var columns
+- **view_data** - View slices of any attribute (obs, var, X, obsm, uns, etc.)
+- **plot_embedding** - UMAP/PCA scatter plots as base64 PNG
+- **get_cap_annotations** - Inspect CAP cell annotation metadata
+

hca_anndata_tools-0.2.0/README.md

@@ -0,0 +1,34 @@
+# hca-anndata-tools
+
+Library for inspection, summarization, and statistics of AnnData h5ad files.
+
+## Installation
+
+```bash
+pip install hca-anndata-tools
+```
+
+## Usage
+
+```python
+from hca_anndata_tools import get_summary, get_descriptive_stats, locate_files
+
+# Find h5ad files
+files = locate_files("/path/to/data")
+
+# Get structural overview
+summary = get_summary("/path/to/my_atlas.h5ad")
+
+# Get column statistics
+stats = get_descriptive_stats("/path/to/my_atlas.h5ad", columns=["cell_type"], value_counts=True)
+```
+
+## API
+
+- **locate_files** - Find all .h5ad files in a directory
+- **get_summary** - Structural overview: cell/gene counts, columns, embeddings, layers
+- **get_storage_info** - HDF5 compression, chunking, sparse format details
+- **get_descriptive_stats** - Descriptive statistics and value counts for obs/var columns
+- **view_data** - View slices of any attribute (obs, var, X, obsm, uns, etc.)
+- **plot_embedding** - UMAP/PCA scatter plots as base64 PNG
+- **get_cap_annotations** - Inspect CAP cell annotation metadata

hca_anndata_tools-0.2.0/pyproject.toml

@@ -0,0 +1,26 @@
+[tool.poetry]
+name = "hca-anndata-tools"
+version = "0.2.0"
+description = "Library for inspection, summarization, and statistics of AnnData h5ad files"
+authors = ["Clever Canary <dave@clevercanary.com>"]
+license = "MIT"
+readme = "README.md"
+homepage = "https://github.com/clevercanary/hca-validation-tools"
+packages = [{include = "hca_anndata_tools", from = "src"}]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+anndata = ">=0.10,<1"
+pandas = ">2,<3"
+numpy = "<3"
+h5py = ">=3.8"
+scanpy = ">=1.10"
+matplotlib = ">=3.7"
+pydantic = ">=2,<3"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

hca_anndata_tools-0.2.0/src/hca_anndata_tools/__init__.py

@@ -0,0 +1,49 @@
+"""HCA AnnData Tools — inspection, summarization, and statistics for h5ad files."""
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "locate_files",
+    "get_summary",
+    "get_descriptive_stats",
+    "view_data",
+    "get_storage_info",
+    "get_cap_annotations",
+    "plot_embedding",
+    "write_h5ad",
+    "strip_timestamp",
+    "generate_output_path",
+    "generate_timestamp",
+    "EDIT_LOG_KEY",
+    "resolve_latest",
+    "set_uns",
+    "list_uns_fields",
+    "convert_cellxgene_to_hca",
+]
+
+_LAZY_IMPORTS = {
+    "locate_files": ".files",
+    "get_summary": ".summary",
+    "get_descriptive_stats": ".stats",
+    "view_data": ".view",
+    "get_storage_info": ".storage",
+    "get_cap_annotations": ".cap",
+    "plot_embedding": ".plot",
+    "write_h5ad": ".write",
+    "strip_timestamp": ".write",
+    "generate_output_path": ".write",
+    "generate_timestamp": ".write",
+    "EDIT_LOG_KEY": ".write",
+    "resolve_latest": ".write",
+    "set_uns": ".edit",
+    "list_uns_fields": ".edit",
+    "convert_cellxgene_to_hca": ".convert",
+}
+
+
+def __getattr__(name: str):
+    if name in _LAZY_IMPORTS:
+        import importlib
+        module = importlib.import_module(_LAZY_IMPORTS[name], __name__)
+        return getattr(module, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

hca_anndata_tools-0.2.0/src/hca_anndata_tools/_io.py

@@ -0,0 +1,27 @@
+"""Internal I/O utilities for AnnData file access."""
+
+import gc
+from contextlib import contextmanager
+
+import anndata as ad
+
+
+@contextmanager
+def open_h5ad(path: str, backed: str | None = "r"):
+    """Open an h5ad file with automatic cleanup.
+
+    Args:
+        path: Absolute path to an .h5ad file.
+        backed: Backing mode. Use "r" for read-only (default), None for full in-memory read.
+
+    Yields:
+        An AnnData object.
+    """
+    adata = ad.read_h5ad(path, backed=backed)
+    try:
+        yield adata
+    finally:
+        if hasattr(adata, "file") and adata.file is not None:
+            adata.file.close()
+        del adata
+        gc.collect()

hca_anndata_tools-0.2.0/src/hca_anndata_tools/_serialize.py

@@ -0,0 +1,24 @@
+"""Shared serialization utilities for converting numpy types to JSON-serializable Python types."""
+
+import numpy as np
+
+
+def make_serializable(obj):
+    """Recursively convert numpy types to JSON-serializable Python types."""
+    if isinstance(obj, (np.integer,)):
+        return int(obj)
+    if isinstance(obj, (np.floating,)):
+        return float(obj)
+    if isinstance(obj, (np.bool_,)):
+        return bool(obj)
+    if isinstance(obj, (np.str_, np.bytes_)):
+        return str(obj)
+    if isinstance(obj, np.ndarray):
+        return obj.tolist()
+    if isinstance(obj, bytes):
+        return obj.decode("utf-8", errors="replace")
+    if isinstance(obj, dict):
+        return {k: make_serializable(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [make_serializable(v) for v in obj]
+    return obj

hca_anndata_tools-0.2.0/src/hca_anndata_tools/cap.py

@@ -0,0 +1,162 @@
+"""CAP (Cell Annotation Platform) schema inspection for AnnData files."""
+
+from ._io import open_h5ad
+from ._serialize import make_serializable as _make_serializable
+
+# CAP obs column suffixes per the cell-annotation-schema spec
+_REQUIRED_SUFFIXES = [
+    "",  # the cell label column itself
+    "--cell_fullname",
+    "--cell_ontology_exists",
+    "--cell_ontology_term_id",
+    "--cell_ontology_term",
+    "--rationale",
+    "--marker_gene_evidence",
+]
+
+_OPTIONAL_SUFFIXES = [
+    "--rationale_dois",
+    "--canonical_marker_genes",
+    "--synonyms",
+    "--category_fullname",
+    "--category_cell_ontology_term_id",
+    "--category_cell_ontology_term",
+    "--cell_ontology_assessment",
+    "--confidence_score",
+]
+
+_UNS_REQUIRED_KEYS = [
+    "cellannotation_schema_version",
+    "cellannotation_metadata",
+    "title",
+]
+
+_UNS_OPTIONAL_KEYS = [
+    "publication_timestamp",
+    "publication_version",
+    "description",
+    "cap_dataset_url",
+    "cap_publication_title",
+    "cap_publication_description",
+    "cap_publication_url",
+    "authors_list",
+    "hierarchy",
+]
+
+
+def _find_annotation_sets(obs_columns: list[str]) -> list[str]:
+    """Identify CAP annotation set names from obs columns with -- separator."""
+    sets = set()
+    for col in obs_columns:
+        if "--" in col:
+            setname = col.split("--")[0]
+            sets.add(setname)
+    return sorted(sets)
+
+
+def _get_unique_values(series, max_values: int = 50) -> list:
+    """Get unique values from a series, capped at max_values."""
+    uniques = [_make_serializable(v) for v in series.dropna().unique().tolist()]
+    if len(uniques) > max_values:
+        return uniques[:max_values] + [f"... and {len(uniques) - max_values} more"]
+    return uniques
+
+
+def get_cap_annotations(path: str, annotation_set: str | None = None) -> dict:
+    """Inspect CAP (Cell Annotation Platform) annotations in an AnnData file.
+
+    Detects CAP-formatted annotation sets, lists cell labels with their ontology
+    mappings, marker genes, and rationale. Reports schema compliance.
+
+    Args:
+        path: Absolute path to an .h5ad file.
+        annotation_set: Specific annotation set name to inspect. If None, lists all sets found.
+    """
+    try:
+        with open_h5ad(path) as adata:
+            obs_columns = list(adata.obs.columns)
+            annotation_sets = _find_annotation_sets(obs_columns)
+
+            uns_keys = list(adata.uns.keys())
+            uns_present = [k for k in _UNS_REQUIRED_KEYS if k in uns_keys]
+            uns_missing = [k for k in _UNS_REQUIRED_KEYS if k not in uns_keys]
+            uns_optional_present = [k for k in _UNS_OPTIONAL_KEYS if k in uns_keys]
+
+            has_cap = len(annotation_sets) > 0 or "cellannotation_metadata" in uns_keys
+
+            result = {
+                "has_cap_annotations": has_cap,
+                "annotation_sets": annotation_sets,
+                "uns_metadata": {
+                    "required_present": uns_present,
+                    "required_missing": uns_missing,
+                    "optional_present": uns_optional_present,
+                },
+            }
+
+            if not has_cap and annotation_set is None:
+                return result
+
+            if "cellannotation_metadata" in uns_keys:
+                result["cellannotation_metadata"] = _make_serializable(adata.uns["cellannotation_metadata"])
+
+            if "authors_list" in uns_keys:
+                result["authors_list"] = _make_serializable(adata.uns["authors_list"])
+
+            sets_to_inspect = []
+            if annotation_set:
+                if annotation_set in annotation_sets:
+                    sets_to_inspect = [annotation_set]
+                else:
+                    return {"error": f"Annotation set '{annotation_set}' not found. Available: {annotation_sets}"}
+            elif len(annotation_sets) <= 3:
+                sets_to_inspect = annotation_sets
+
+            set_details = {}
+            for setname in sets_to_inspect:
+                detail = {}
+
+                required_found = []
+                required_missing = []
+                for suffix in _REQUIRED_SUFFIXES:
+                    col = f"{setname}{suffix}" if suffix else setname
+                    if col in obs_columns:
+                        required_found.append(col)
+                    else:
+                        required_missing.append(col)
+
+                optional_found = []
+                for suffix in _OPTIONAL_SUFFIXES:
+                    col = f"{setname}{suffix}"
+                    if col in obs_columns:
+                        optional_found.append(col)
+
+                detail["required_columns_present"] = required_found
+                detail["required_columns_missing"] = required_missing
+                detail["optional_columns_present"] = optional_found
+
+                if setname in obs_columns:
+                    vc = adata.obs[setname].value_counts()
+                    detail["cell_labels"] = {str(k): int(v) for k, v in vc.items()}
+
+                ont_col = f"{setname}--cell_ontology_term_id"
+                if ont_col in obs_columns:
+                    detail["ontology_terms"] = _get_unique_values(adata.obs[ont_col])
+
+                marker_col = f"{setname}--marker_gene_evidence"
+                if marker_col in obs_columns:
+                    detail["marker_genes"] = _get_unique_values(adata.obs[marker_col])
+
+                rat_col = f"{setname}--rationale"
+                if rat_col in obs_columns:
+                    detail["rationale"] = _get_unique_values(adata.obs[rat_col])
+
+                set_details[setname] = detail
+
+            if set_details:
+                result["set_details"] = set_details
+
+            return result
+
+    except Exception as e:
+        return {"error": str(e)}

hca_anndata_tools-0.2.0/src/hca_anndata_tools/convert.py

@@ -0,0 +1,136 @@
+"""Convert CellxGENE h5ad files to HCA schema format."""
+
+from __future__ import annotations
+
+import os
+import re
+from datetime import datetime, timezone
+
+import numpy as np
+import pandas as pd
+
+from ._io import open_h5ad
+from ._serialize import make_serializable
+from .write import generate_timestamp, write_h5ad
+from . import __version__
+
+# CellxGENE reserved uns keys — moved to cellxgene_source, not deleted
+_CELLXGENE_RESERVED_UNS = ["schema_version", "schema_reference", "citation"]
+
+# CellxGENE uns keys that need to be broadcast to obs
+_UNS_TO_OBS = ["organism_ontology_term_id", "organism"]
+
+
+def _slugify(text: str, max_length: int = 80) -> str:
+    """Convert text to a filename-safe slug."""
+    slug = text.lower()
+    slug = re.sub(r"[^a-z0-9]+", "-", slug)
+    slug = slug.strip("-")
+    if len(slug) > max_length:
+        slug = slug[:max_length].rstrip("-")
+    return slug or "untitled"
+
+
+def convert_cellxgene_to_hca(
+    path: str,
+    output_dir: str | None = None,
+) -> dict:
+    """Convert a CellxGENE h5ad file to HCA schema format.
+
+    Preserves cellxgene provenance in uns['cellxgene_source'], broadcasts
+    organism from uns to obs, and renames the output from the dataset title.
+    Does not add missing HCA-specific fields — wranglers fill those later.
+
+    Args:
+        path: Path to a CellxGENE h5ad file.
+        output_dir: Directory for the output file. Defaults to same as source.
+
+    Returns:
+        Dict with 'output_path', 'source', 'title', 'conversions' on success,
+        or 'error' on failure.
+    """
+    try:
+        with open_h5ad(path, backed=None) as adata:
+            # Verify this is a cellxgene 6.0+ file (organism in uns, single species)
+            if "schema_version" not in adata.uns:
+                return {"error": "Not a CellxGENE file — uns['schema_version'] is missing"}
+
+            schema_ver = str(adata.uns["schema_version"]).strip()
+            if not schema_ver:
+                return {"error": "uns['schema_version'] is empty — cannot determine CellxGENE schema version"}
+            major = int(schema_ver.split(".")[0]) if schema_ver[0].isdigit() else 0
+            if major < 6:
+                return {
+                    "error": (
+                        f"CellxGENE schema {schema_ver} is not supported. "
+                        f"Requires 6.0+ (organism in uns, single species per dataset)."
+                    )
+                }
+
+            title = adata.uns.get("title", "")
+            if not title:
+                return {"error": "File has no title in uns — cannot generate output filename"}
+
+            conversions = []
+
+            # --- 1. Preserve cellxgene provenance ---
+            cellxgene_source = {}
+            for key in _CELLXGENE_RESERVED_UNS:
+                if key in adata.uns:
+                    cellxgene_source[key] = make_serializable(adata.uns[key])
+                    del adata.uns[key]
+
+            if cellxgene_source:
+                adata.uns["cellxgene_source"] = cellxgene_source
+                conversions.append(
+                    f"Moved cellxgene reserved keys to uns['cellxgene_source']: "
+                    f"{list(cellxgene_source.keys())}"
+                )
+
+            # --- 2. Broadcast organism from uns → obs ---
+            for key in _UNS_TO_OBS:
+                if key in adata.uns:
+                    value = adata.uns[key]
+                    adata.obs[key] = pd.Categorical.from_codes(
+                        np.zeros(adata.n_obs, dtype=np.int8),
+                        categories=[value],
+                    )
+                    del adata.uns[key]
+                    conversions.append(f"{key}: uns → obs (broadcast '{value}')")
+
+            # --- 3. Build output path from title slug ---
+            slug = _slugify(title)
+            timestamp = generate_timestamp()
+            out_filename = f"{slug}-edit-{timestamp}.h5ad"
+            directory = output_dir if output_dir is not None else os.path.dirname(path)
+            output_path = os.path.join(directory, out_filename)
+
+            # --- 4. Build edit log entry ---
+            entry = {
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "tool": "hca-anndata-tools",
+                "tool_version": __version__,
+                "operation": "import_cellxgene",
+                "description": f"Imported from CellxGENE Discover: {title}",
+                "details": {
+                    "source_schema_version": cellxgene_source.get("schema_version"),
+                    "source_citation": cellxgene_source.get("citation"),
+                    "conversions": conversions,
+                },
+            }
+
+            # --- 5. Write ---
+            result = write_h5ad(adata, path, [entry], output_path=output_path)
+
+        if "error" in result:
+            return result
+
+        return {
+            **result,
+            "source": os.path.basename(path),
+            "title": title,
+            "conversions": conversions,
+        }
+
+    except Exception as e:
+        return {"error": str(e)}