hca-schema-validator 0.2.0__py3-none-any.whl

hca_schema_validator/_vendored/cellxgene_schema/map_species.py

@@ -0,0 +1,40 @@
+import logging
+
+import anndata as ad
+from cellxgene_ontology_guide.ontology_parser import OntologyParser
+
+logger = logging.getLogger(__name__)
+
+
+def map_species(input_file, output_file):
+    ontology_parser = OntologyParser("6.0.0")
+    adata = ad.read_h5ad(input_file, backed="r")
+    map_columns = [
+        ("organism_cell_type_ontology_term_id", "cell_type_ontology_term_id", "CL"),
+        ("organism_tissue_ontology_term_id", "tissue_ontology_term_id", "UBERON"),
+    ]
+    for map_from, map_to, ontology in map_columns:
+        logger.info(f"Mapping {ontology} terms to {map_to}")
+        if not hasattr(adata.obs, map_to):
+            logger.info(f"No existing {map_to} column detected, creating one and defaulting to empty strings.")
+            adata.obs[map_to] = ""
+        for category, group in adata.obs.groupby(map_from):
+            try:
+                closest_terms = ontology_parser.get_closest_bridge_term_ids(category, ontology)
+            except Exception:
+                logger.error(f"{category} in {map_from} is not a valid ontology term ID, cannot find {ontology} map.")
+                continue
+            if len(closest_terms) < 1:
+                logger.warning(f"{category} has no closest match for {ontology} in its ancestry tree.")
+            elif len(closest_terms) == 1:
+                adata.obs.loc[group.index, map_to] = closest_terms[0]
+                logger.info(f"Setting {category} {map_to} rows to single closest match: {closest_terms[0]}")
+            else:
+                logger.warning(
+                    f"{category} has multiple closest matches for {ontology} in its ancestry tree: "
+                    f"{closest_terms}. Pick one based on biological context and assign manually."
+                )
+        adata.obs[map_to] = adata.obs[map_to].astype("category")
+
+    logger.info(f"Mappings complete, writing to {output_file}")
+    adata.write_h5ad(output_file, compression="gzip")

hca_schema_validator/_vendored/cellxgene_schema/matrix_utils.py

@@ -0,0 +1,134 @@
+from typing import Union
+
+import anndata as ad
+import numpy as np
+import sparse as sp
+from anndata.compat import DaskArray
+from dask.array import map_blocks
+from scipy import sparse
+
+SPARSE_MATRIX_TYPES = {"csr", "csc", "coo"}
+SUPPORTED_SPARSE_MATRIX_TYPES = {"csr"}
+
+
+def determine_matrix_format(matrix: DaskArray) -> str:
+    """
+    Given a matrix, returns the format as one of: csc, csr, coo, dense
+    or unknown.
+
+    This mimics the scipy.sparse `format` property, but extends it to
+    support ndarray and other classes AnnData may proxy the matrix with.
+    """
+
+    # Note: the AnnData proxy classes DO support the `format_str` property, but
+    # doing a slice seemed safer, if less performant.  Using `format_str`, which
+    # currently works, uses private API:
+    #
+    # >>> return getattr(matrix, "format_str", "dense)
+    #
+    matrix_format = "unknown"
+    try:
+        matrix_slice = matrix[0:1, 0:1].compute()
+    except AttributeError:
+        # compute() may fail on an unknown matrix value. if so, return "unknown"
+        return matrix_format
+    if isinstance(matrix_slice, sparse.spmatrix):
+        matrix_format = matrix_slice.format
+    elif isinstance(matrix_slice, np.ndarray):
+        matrix_format = "dense"
+    assert matrix_format in SPARSE_MATRIX_TYPES.union({"unknown", "dense"})
+    return matrix_format
+
+
+def compute_column_sums(matrix: Union[DaskArray, np.ndarray, sparse.spmatrix]) -> np.ndarray:
+    """
+    Compute column-wise sums for a Dask array (dense or sparse), NumPy ndarray, or SciPy sparse matrix.
+    Returns a NumPy array of sums.
+
+    For example, this matrix:
+    [
+        [1, 0, 0],
+        [1, 2, 0],
+        [1, 0, 1],
+    ]
+
+    would return a NumPy array of [3, 12, 1]
+    """
+    if isinstance(matrix, np.ndarray):
+        return matrix.sum(axis=0).ravel()
+
+    if sparse.issparse(matrix):
+        return np.array(matrix.sum(axis=0)).ravel()
+
+    # Handle Dask array (could be sparse or dense)
+    if isinstance(matrix, DaskArray):
+
+        def to_coo(chunk: Union[np.ndarray, sparse.spmatrix]) -> Union[sp.COO, np.ndarray]:
+            # convert to sparse.COO, so we can use dask's internal chunked sum
+            if sparse.issparse(chunk):
+                return sp.COO.from_scipy_sparse(chunk)
+            else:
+                return chunk
+
+        x_coo = matrix.map_blocks(to_coo, dtype=matrix.dtype)
+
+        # 'split_every' is used to ensure that the computation is split into memory manageable chunks
+        col_sums = x_coo.sum(axis=0, split_every=8).compute()
+        return col_sums
+
+
+def calculate_matrix_nonzero(matrix: DaskArray) -> int:
+    def count_nonzeros(matrix_chunk: Union[np.ndarray, sparse.spmatrix], is_sparse_matrix: bool) -> np.array:
+        nnz = matrix_chunk.nnz if is_sparse_matrix else np.count_nonzero(matrix_chunk)
+        return np.array([nnz])
+
+    is_sparse_matrix = determine_matrix_format(matrix) in SPARSE_MATRIX_TYPES
+    if len(matrix.chunks[0]) > 1:
+        nonzeros = map_blocks(count_nonzeros, matrix, is_sparse_matrix, drop_axis=1, dtype=int).compute().sum()
+    else:
+        nonzeros = count_nonzeros(matrix.compute(), is_sparse_matrix)[0]
+    return nonzeros
+
+
+def check_non_csr_matrixes(adata: ad.AnnData):
+    """
+    Check X, raw.X and layers matrices for having more than 50% zeros and not being csr_matrix
+
+    If found, convert to csr_matrix
+    """
+
+    def get_sparsity(matrix: DaskArray, format: str):
+        is_sparse_matrix = format in SPARSE_MATRIX_TYPES
+        nnz = calculate_matrix_nonzero(matrix, is_sparse_matrix)
+        sparsity = 1 - nnz / np.prod(matrix.shape)
+        return sparsity
+
+    format = determine_matrix_format(adata.X)
+    if format != "csr" and get_sparsity(adata.X, format) >= 0.5:
+        adata.X = adata.X.map_blocks(sparse.csr_matrix, dtype=adata.X.dtype)
+
+    if adata.raw is not None:
+        format = determine_matrix_format(adata.raw.X)
+        if format != "csr" and get_sparsity(adata.raw.X, format) >= 0.5:
+            raw_adata = ad.AnnData(adata.raw.X, var=adata.raw.var, obs=adata.obs)
+            raw_adata.X = raw_adata.X.map_blocks(sparse.csr_matrix, dtype=raw_adata.X.dtype)
+            adata.raw = raw_adata
+            del raw_adata
+
+    for layer in adata.layers:
+        format = determine_matrix_format(adata.layers[layer])
+        if format != "csr" and get_sparsity(adata.layers[layer], format) >= 0.5:
+            adata.layers[layer] = adata.layers[layer].map_blocks(sparse.csr_matrix, dtype=adata.layers[layer].X.dtype)
+
+    return adata
+
+
+def debug_print_matrix(matrix: ad.AnnData, matrix_name: str, max_rows=20, max_cols=20):
+    computed = matrix[:max_rows, :max_cols].compute()
+
+    # If sparse, convert to dense for display
+    if hasattr(computed, "toarray"):
+        computed = computed.toarray()
+
+    print(f"{matrix_name} (first {max_rows} rows, {max_cols} cols):")
+    print(np.round(computed, 2))  # round for readability

hca_schema_validator/_vendored/cellxgene_schema/migrate.py

@@ -0,0 +1,145 @@
+import dask
+
+from . import utils
+
+# fmt: off
+# ONTOLOGY TERMS TO UPDATE ACROSS ALL DATASETS IN CORPUS
+# Initialization is AUTOMATED for newly deprecated terms that have 'Replaced By' terms in their ontology files
+
+# Curators should review the monthly 'Curator Report' and add deprecated term replacements to corresponding map if
+# 'Replaced By' is not available for a deprecated term.
+
+# If Curators have non-deprecated term changes to apply to all datasets in the corpus where applicable,
+# add them here.
+# added two CL terms here instead of as applicable to individual datasets
+ONTOLOGY_TERM_OBS_MAPS = {
+    "assay": {
+    },
+    "cell_type": {
+        "CL:0000651": "CL:0002181",
+        "CL:0000212": "CL:0000677"
+    },
+    "development_stage": {
+    },
+    "disease": {
+    },
+    "self_reported_ethnicity": {
+    },
+    "sex": {
+    },
+    "tissue": {
+    },
+}
+
+ONTOLOGY_TERM_UNS_MAPS = {
+    "organism": {
+    },
+}
+
+DEPRECATED_FEATURE_IDS = [
+]
+
+# Dictionary for CURATOR-DEFINED remapping of deprecated feature IDs, if any, to new feature IDs.
+GENCODE_MAPPER = {}
+# fmt: on
+
+
+def migrate(input_file, output_file, collection_id, dataset_id):
+    print(f"Converting {input_file} into {output_file}")
+
+    dataset = utils.read_h5ad(input_file)
+
+    # Migrate organism_ontology_term_id for 6.0.0
+    dataset = utils.move_column_from_obs_to_uns(adata=dataset, column_name="organism_ontology_term_id")
+
+    # Migrate delimter from `,` to ` || ` for self_reported_ethnicity_ontology_term_id
+    utils.replace_delimiter(
+        dataframe=dataset.obs,
+        old_delimiter=",",
+        new_delimiter=" || ",
+        column_name="self_reported_ethnicity_ontology_term_id",
+    )
+
+    # AUTOMATED, DO NOT CHANGE
+    for ontology_name, deprecated_term_map in ONTOLOGY_TERM_OBS_MAPS.items():
+        utils.replace_ontology_term(dataset.obs, ontology_name, deprecated_term_map)
+    for ontology_name, deprecated_term_map in ONTOLOGY_TERM_UNS_MAPS.items():
+        dataset = utils.replace_ontology_term_uns(dataset, ontology_name, deprecated_term_map)
+
+    # CURATOR-DEFINED, DATASET-SPECIFIC UPDATES
+    # Use the template below to define dataset and collection specific ontology changes. Will only apply to dataset
+    # if it matches a condition.
+    # If no such changes are needed, leave blank
+    # Examples:
+    # if dataset_id == "<dataset_1_id>":
+    #   <no further logic necessary>
+    #   utils.replace_ontology_term(df, <ontology_name>, {"term_to_replace": "replacement_term", ...})
+    # elif dataset_id == "<dataset_2_id>":
+    #   <custom transformation logic beyond scope of util functions>
+    # elif collection_id == "<collection_1_id>":
+    #   <no further logic necessary>
+    #   utils.replace_ontology_term(df, <ontology_name>, {"term_to_replace": "replacement_term", ...})
+    # elif collection_id == "<collection_2_id>":
+    #   <custom transformation logic beyond scope of replace_ontology_term>
+    # ...
+
+    if (
+        dataset.uns["title"]
+        == "Time-resolved single-cell analysis of Brca1 associated mammary tumourigenesis reveals aberrant differentiation of luminal progenitors"
+    ):
+        utils.replace_ontology_term(dataset.obs, "cell_type", {"CL:0000451": "CL:4047054"})
+
+    if collection_id == "126afc71-47fb-4e9d-8aaf-9d9f61e0ac77":
+        utils.map_ontology_term(
+            dataset.obs,
+            "disease",
+            "disease_group",
+            {
+                "Healthy": "PATO:0000461",
+                "HIV": "MONDO:0005109",
+                "VL_HIV": "MONDO:0005109 || MONDO:0005445",
+                "aL_HIV": "MONDO:0005109 || MONDO:0011989",
+                "cVL_HIV": "MONDO:0005109 || MONDO:0005445",
+                "pVL_HIV": "MONDO:0005109 || MONDO:0005445",
+            },
+        )
+
+    if collection_id == "eeba7193-4d32-46bd-a21b-089936d60601":
+        utils.map_ontology_term(
+            dataset.obs,
+            "disease",
+            "sample_id",
+            {
+                "mraf1": "MONDO:0004981 || MONDO:1030008",
+                "mraf2": "MONDO:0004981 || MONDO:1030008",
+                "mraf3": "MONDO:0004981 || MONDO:1030008",
+                "mraf4": "MONDO:0004981 || MONDO:1030008",
+                "mraf7": "MONDO:0004981 || MONDO:1030008",
+                "mraf13": "MONDO:0004981 || MONDO:1030008",
+                "mraf14": "MONDO:0004981 || MONDO:1030008",
+            },
+        )
+
+    # errors found in 3 dev migration runs
+    # will just check for key and delete instead of list of dataset_ids
+    if "organism" in dataset.uns:
+        del dataset.uns["organism"]
+
+    # feature_is_filtered set incorrectly for dataset with no raw.X matrix, should be all False
+    if dataset_id == "ee195b7d-184d-4dfa-9b1c-51a7e601ac11":
+        dataset.var["feature_is_filtered"] = False
+
+    # 11 datasets with "organism_ontology_term_id_colors" in uns, all can be removed
+    # seems easy enough to quickly check for key, can also list out the 11 dataset_ids
+    if "organism_ontology_term_id_colors" in dataset.uns:
+        del dataset.uns["organism_ontology_term_id_colors"]
+
+    if GENCODE_MAPPER:
+        dataset = utils.remap_deprecated_features(adata=dataset, remapped_features=GENCODE_MAPPER)
+
+    # AUTOMATED, DO NOT CHANGE -- IF GENCODE UPDATED, DEPRECATED FEATURE FILTERING ALGORITHM WILL GO HERE.
+    if DEPRECATED_FEATURE_IDS:
+        dataset = utils.remove_deprecated_features(adata=dataset, deprecated=DEPRECATED_FEATURE_IDS)
+
+    with dask.config.set(scheduler="single-threaded"):
+        dataset.write_h5ad(output_file, compression="gzip")

hca_schema_validator/_vendored/cellxgene_schema/migrate_files/automigrate_terms.json

@@ -0,0 +1,2 @@
+{
+}

hca_schema_validator/_vendored/cellxgene_schema/migrate_files/donor_updates.json

@@ -0,0 +1,2 @@
+{
+}

hca_schema_validator/_vendored/cellxgene_schema/migrate_files/non_csr_list.txt

@@ -0,0 +1,54 @@
+7f7faf6b-f11d-4f07-bc1c-188a4472748d
+7c497c66-c2e2-4291-8b96-339e5e65f314
+7b55fe5c-d5c8-48e0-a1a3-54c5b9074f3f
+2ac76f1b-43ef-4271-8686-2f165570989f
+f9cfac8d-bff6-47a2-a1f6-503827d375f5
+ea95cdd5-cf7e-4d5d-a0bd-032776d32ae5
+e34f3f89-3a48-4173-a06a-9e9803f2e2b9
+e2a325e2-5633-4950-ae46-ebff70979e3b
+d5a2d011-3765-4dad-ac36-3fff087589fe
+b6e502d7-0d07-408e-a291-91b63af0a40a
+a30db232-08a6-48f0-ab9f-ad6977b8a6b1
+9ea52f85-a35e-4b5b-9207-e2f1de7dbbcd
+93091496-be48-4122-b945-9af9c22a7535
+78f40dd6-ed22-470d-8547-6fb5746c25da
+6929d6d1-6f81-44d7-bd35-5a580259af81
+5e7fff82-b8eb-4220-bf65-c6feb6765ea1
+4e374783-0152-423e-bcb6-b87cc232f64d
+4859642c-d155-437b-adf5-83ca6c1bef67
+390b376d-8857-4b4b-991f-0d7326938172
+34229bdc-a895-4394-8820-574e4028d8c6
+31285bcd-9a28-48ce-8cc0-543af80261d3
+3079e9b0-c6db-45c8-b998-a4555a73968e
+1fc48734-35ee-4372-885c-95fd2674c75a
+1a974af7-8b4b-4abb-8aba-23e2979d9518
+171b03c9-516b-499c-a434-fa4f70191988
+149b2c3f-ee11-47a7-984b-923570280bd7
+0de831e0-a525-4ec5-b717-df56f2de2bf0
+0087cde2-967d-4f7c-8e6e-40e4c9ad1891
+e04cb7d9-7487-4bf7-99db-b16f236a351a
+b2dd6bc9-36ed-4cd4-a2b3-51f6a2386afa
+a19d1667-a7b5-4556-9e5f-f9bfa690c0f1
+510151f7-0cf7-4679-a15b-b7cd824f19bb
+30a2789f-7b76-4414-9c63-1deb3eacdde1
+20de7fd1-ab21-45b8-a1e2-f7b36261f8e2
+f0939ae1-0877-42b2-bc0e-3249a42c2c0f
+e8d1c9cc-29e6-4b51-90f1-29e75de6098f
+d6f3c096-a6f0-4e67-91ad-514bd286f13e
+a12d95f7-480f-4c8a-bf24-302175b27bc4
+8f83f50e-c091-4095-8456-a7acf6c9d423
+52f0b078-0fb7-4cdb-81f8-d1bcf7b4468b
+427bfbe6-1141-4505-98e3-38aa05339310
+3ed61f1e-bd31-411e-b4aa-9729e8437732
+38e5738d-3870-433b-858a-5c364f322439
+346c5aad-b034-4248-8cbe-0a05fd634b9c
+33da10b0-9c1d-4c82-9b14-c67cdcf9fae5
+0500e103-38db-456d-9c3f-b96b8a693ab2
+cb34cb2d-aee2-4272-86ed-f8e1af870e52
+c20f1b97-0d47-4192-af0a-1e012621f8d9
+3bbb6cf9-72b9-41be-b568-656de6eb18b5
+e40c6272-af77-4a10-9385-62a398884f27
+d6dfdef1-406d-4efb-808c-3c5eddbfe0cb
+6a270451-b4d9-43e0-aa89-e33aac1ac74b
+c888b684-6c51-431f-972a-6c963044cef0
+ae4f8ddd-cac9-4172-9681-2175da462f2e

hca_schema_validator/_vendored/cellxgene_schema/migrate_files/private_non_csr.json

@@ -0,0 +1 @@
+[["Single-cell characterization of the stress response in the adult ventral hippocampus", "Ctrl_401"], ["Single-cell RNA sequencing of human intestinal blood endothelial cells", "donor_ulxdkr"], ["Kidney-injury enriched MKA", "Kirita20"]]

hca_schema_validator/_vendored/cellxgene_schema/migrate_files/title_donor_updates.json

@@ -0,0 +1,2 @@
+{
+}

hca_schema_validator/_vendored/cellxgene_schema/ontology_files/.gitignore

@@ -0,0 +1 @@
+*owl

hca_schema_validator/_vendored/cellxgene_schema/ontology_parser.py

@@ -0,0 +1,3 @@
+from cellxgene_ontology_guide.ontology_parser import OntologyParser
+
+ONTOLOGY_PARSER = OntologyParser(schema_version="v7.0.0")

hca_schema_validator/_vendored/cellxgene_schema/remove_labels.py

@@ -0,0 +1,74 @@
+import logging
+
+import anndata as ad
+from pandas import DataFrame
+
+from . import schema
+from .utils import getattr_anndata
+
+logger = logging.getLogger(__name__)
+
+
+class AnnDataLabelRemover:
+    """
+    From valid h5ad, handles writing a new h5ad file with appended ontology/gene labels removed
+    from adata.obs, adata.var, and adata.raw.var respectively as indicated in the schema definition
+    """
+
+    def __init__(self, adata: ad.AnnData = None):
+        self.adata = adata
+        self.schema_def = schema.get_schema_definition()
+
+    def remove_labels(self):
+        """
+        From a valid (per cellxgene's schema) adata, this function parses the associated schema definition for
+        'add_labels.to_column' annotations, and removes specified columns from self.adata
+        """
+        for component_name in ["obs", "var", "raw.var", "uns"]:
+            # If the component does not exist, skip (this is for raw.var)
+            component = getattr_anndata(self.adata, component_name)
+            if component is None:
+                continue
+
+            component_def = self.schema_def["components"][component_name]
+
+            # Doing it for columns
+            if "columns" in component_def:
+                for column_def in component_def["columns"].values():
+                    if "add_labels" in column_def:
+                        self._remove_columns(component, column_def)
+
+            # Remove automatically annotated columns
+            if "reserved_columns" in component_def:
+                for field in component_def["reserved_columns"]:
+                    del component[field]
+
+            # Doing it for index
+            if "index" in component_def:
+                index_def = component_def["index"]
+                if "add_labels" in index_def:
+                    self._remove_columns(component, index_def)
+
+            # Remove any labels added as dict keys
+            if "keys" in component_def:
+                for key in component_def["keys"]:
+                    key_def = component_def["keys"][key]
+                    if "add_labels" in key_def:
+                        key_to_remove = key_def["add_labels"][0]["to_key"]
+                        del component[key_to_remove]
+
+    def _remove_columns(self, component: DataFrame, subcomponent_definition: dict):
+        """
+        Given an adata component and subcomponent definition, this function deletes all existing columns in the
+        self.adata component that are defined as added labels ('add_labels.to_column') in the subcomponent definition.
+
+        :param pd.Dataframe component: dataframe within adata dataset (i.e. 'obs', 'var', 'raw.var')
+        :param dict subcomponent_definition: yaml-defined schema for subcomponent of component (i.e. index, or a
+                    particular column)
+
+        :rtype None
+        """
+        for label_def in subcomponent_definition["add_labels"]:
+            column_name = label_def["to_column"]
+            if column_name in component:
+                del component[column_name]

hca_schema_validator/_vendored/cellxgene_schema/schema.py

@@ -0,0 +1,21 @@
+import semver
+import yaml
+
+from . import __schema_version__, env
+
+
+def get_schema_definition() -> dict:
+    """
+    Look up and read the current schema definition
+    :return The schema definition
+    :rtype dict
+    """
+
+    path = env.SCHEMA_DEFINITION_FILE
+    with open(path) as fp:
+        return yaml.load(fp, Loader=yaml.FullLoader)
+
+
+def get_current_schema_version() -> str:
+    current_version: semver.Version = semver.Version.parse(__schema_version__)
+    return f"{str(current_version.major)}.{str(current_version.minor)}.0"