hca-schema-validator 0.2.0__py3-none-any.whl

hca_schema_validator/_vendored/cellxgene_schema/cli.py

@@ -0,0 +1,206 @@
+import logging
+import sys
+
+import click
+
+logger = logging.getLogger("cellxgene_schema")
+
+
+@click.group(
+    name="schema",
+    subcommand_metavar="COMMAND <args>",
+    short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
+    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
+)
+@click.option("-v", "--verbose", help="When present will set logging level to debug", is_flag=True)
+def schema_cli(verbose):
+    logging.basicConfig(level=logging.ERROR)
+    logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+
+
+@schema_cli.command(
+    name="validate",
+    short_help="Check that an h5ad follows the cellxgene data integration schema.",
+    help="Check that an h5ad follows the cellxgene data integration schema. If validation fails this command will "
+    "return an exit status of 1 otherwise 0. When the '--add-labels <FILE>' tag is present, the command will add "
+    "ontology/gene labels based on IDs and write them to a new h5ad.",
+)
+@click.argument("h5ad_file", nargs=1, type=click.Path(exists=True, dir_okay=False))
+@click.option(
+    "-a",
+    "--add-labels",
+    "add_labels_file",
+    help="When present it will add labels to genes and ontologies based on IDs",
+    required=False,
+    default=None,
+    type=click.Path(exists=False, dir_okay=False, writable=True),
+)
+@click.option("-i", "--ignore-labels", help="Ignore ontology labels when validating", is_flag=True)
+def schema_validate(h5ad_file, add_labels_file, ignore_labels):
+    # Imports are very slow so we defer loading until Click arg validation has passed
+    logger.info("Loading dependencies")
+    try:
+        import anndata  # noqa: F401
+    except ImportError:
+        raise click.ClickException("[cellxgene] cellxgene-schema requires anndata") from None
+
+    logger.info("Loading validator modules")
+    from .validate import validate
+
+    is_valid, _, _ = validate(h5ad_file, add_labels_file, ignore_labels=ignore_labels)
+    if is_valid:
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+
+@schema_cli.command(
+    name="add-labels",
+    short_help="Create a copy of an h5ad with portal-added labels",
+    help="Create a copy of an h5ad with portal-added labels. The labels are added based on the IDs in the "
+    "provided file.",
+)
+@click.argument("input_file", nargs=1, type=click.Path(exists=True, dir_okay=False))
+@click.argument("output_file", nargs=1, type=click.Path(exists=False, dir_okay=False))
+def add_labels(input_file, output_file):
+    from utils import read_h5ad
+
+    from .write_labels import AnnDataLabelAppender
+
+    logger.info(f"Loading h5ad from {input_file}")
+    adata = read_h5ad(input_file)
+    anndata_label_adder = AnnDataLabelAppender(adata)
+    logger.info("Adding labels")
+    if anndata_label_adder.write_labels(output_file):
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+
+def _check_anndata_requires_fragment(h5ad_file):
+    from .atac_seq import check_anndata_requires_fragment, report_errors
+
+    try:
+        fragment_required = check_anndata_requires_fragment(h5ad_file)
+        if fragment_required:
+            logger.info("Anndata requires an ATAC fragment file.")
+        else:
+            logger.info("Anndata does not require an ATAC fragment file.")
+    except Exception as e:
+        report_errors("Anndata does not support ATAC fragment files for the followings reasons:", [str(e)])
+        sys.exit(1)
+
+
+@schema_cli.command(
+    name="process-fragment",
+    short_help="Check that an ATAC SEQ fragment follows the cellxgene data integration schema.",
+    help="Check that an ATAC SEQ fragment follows the cellxgene data integration schema. If validation fails this "
+    "command will return an exit status of 1 otherwise 0. When the '--generate-index' tag is present, "
+    "the command will generate a tabix compatible version of the fragment and tabix index. The generated "
+    "fragment will have the file suffix .bgz and the index will have the file suffix .bgz.tbi.",
+)
+@click.argument("h5ad_file", nargs=1, type=click.Path(exists=True, dir_okay=False))
+@click.argument("fragment_file", nargs=1, type=click.Path(exists=True, dir_okay=False))
+@click.option("-i", "--generate-index", help="Generate index for fragment", is_flag=True)
+@click.option("-o", "--output-file", help="Output file for the processed fragment.", type=click.Path(exists=False))
+def fragment_validate(h5ad_file, fragment_file, generate_index, output_file):
+    from .atac_seq import process_fragment
+
+    _check_anndata_requires_fragment(h5ad_file)
+
+    if not process_fragment(fragment_file, h5ad_file, generate_index=generate_index, output_file=output_file):
+        sys.exit(1)
+
+
+# add a cli command to deduplicate an ATAC fragment file
+@schema_cli.command(
+    name="deduplicate-fragment",
+    short_help="Deduplicate an ATAC SEQ fragment file.",
+    help="Deduplicate an ATAC SEQ fragment file. If deduplication fails this command will return an exit status of 1 "
+    "otherwise 0. The deduplicated fragment will have the file suffix .dedup.bgz and the index will have the file "
+    "suffix .dedup.bgz.tbi.",
+)
+@click.argument("fragment_file", nargs=1, type=click.Path(exists=True, dir_okay=False))
+@click.option("-o", "--output-file", help="Output file for the deduplicated fragment.", type=click.Path(exists=False))
+@click.option("-m", "--memory", help="Memory limit as a percentage of total memory.", type=int, default=80)
+def deduplicate_fragment(fragment_file, output_file, memory):
+    from .atac_seq import deduplicate_fragment_rows
+
+    try:
+        deduplicate_fragment_rows(fragment_file, output_file_name=output_file, sort_memory_percent=memory)
+    except Exception as e:
+        logger.error(f"Failed to deduplicate fragment file: {e}")
+        sys.exit(1)
+
+
+@schema_cli.command(
+    name="check-anndata-requires-fragment",
+    short_help="Check if that the anndata provided supports an Atac seq fragment file.",
+    help="Check that an ATAC SEQ anndata.obs['assay_ontology_term_id'] is all paired or unpaired assays. "
+    "This determines if a fragment file is required, optional, or forbiden. "
+    "If the anndata does not support a fragment, an error message will be printed and exit status will be 1.",
+)
+@click.argument("h5ad_file", nargs=1, type=click.Path(exists=True, dir_okay=False))
+def check_anndata_requires_fragment(h5ad_file):
+    _check_anndata_requires_fragment(h5ad_file)
+
+
+@schema_cli.command(
+    name="remove-labels",
+    short_help="Create a copy of an h5ad without portal-added labels",
+    help="Create a copy of an h5ad without portal-added labels.",
+)
+@click.argument("input_file", nargs=1, type=click.Path(exists=True, dir_okay=False))
+@click.argument("output_file", nargs=1, type=click.Path(exists=False, dir_okay=False))
+def remove_labels(input_file, output_file):
+    from .remove_labels import AnnDataLabelRemover
+
+    logger.info("Loading dependencies")
+    try:
+        import anndata  # noqa: F401
+    except ImportError:
+        raise click.ClickException("[cellxgene] cellxgene-schema requires anndata") from None
+
+    logger.info(f"Loading h5ad from {input_file}")
+    adata = anndata.read_h5ad(input_file)
+    anndata_label_remover = AnnDataLabelRemover(adata)
+    if not anndata_label_remover.schema_def:
+        return
+    logger.info("Removing labels")
+    anndata_label_remover.remove_labels()
+    logger.info(f"Labels have been removed. Writing to {output_file}")
+    anndata_label_remover.adata.write(output_file, compression="gzip")
+
+
+@schema_cli.command(
+    name="migrate",
+    short_help="Convert an h5ad to the latest schema version.",
+    help="Convert an h5ad from the previous to latest minor schema version. No validation will be "
+    "performed on either the input or the output file.",
+)
+@click.argument("input_file", nargs=1, type=click.Path(exists=True, dir_okay=False))
+@click.argument("output_file", nargs=1, type=click.Path(exists=False, dir_okay=False))
+@click.option("--collection_id", default=None, type=str, help="Collection ID, if migrating already published dataset")
+@click.option("--dataset_id", default=None, type=str, help="Dataset ID, if migrating already published dataset")
+def migrate(input_file, output_file, collection_id, dataset_id):
+    from .migrate import migrate
+
+    migrate(input_file, output_file, collection_id, dataset_id)
+
+
+@schema_cli.command(
+    name="map-species",
+    short_help="Annotate non-human, non-mouse anndata with CL and UBERON equivalent terms",
+    help="Annotate non-human, non-mouse anndata with CL and UBERON equivalent terms, based on values in"
+    "organism-specific columns (e.g. organism_cell_type_ontology_term_id and organism_tissue_ontology_term_id)",
+)
+@click.argument("input_file", nargs=1, type=click.Path(exists=True, dir_okay=False))
+@click.argument("output_file", nargs=1, type=click.Path(exists=False, dir_okay=False))
+def map_species(input_file, output_file):
+    from .map_species import map_species
+
+    map_species(input_file, output_file)
+
+
+if __name__ == "__main__":
+    schema_cli()

hca_schema_validator/_vendored/cellxgene_schema/env.py

@@ -0,0 +1,9 @@
+import os
+
+PACKAGE_ROOT = os.path.dirname(os.path.realpath(__file__))
+GENCODE_DIR = os.path.join(PACKAGE_ROOT, "gencode_files")
+GENE_INFO_YAML = os.path.join(GENCODE_DIR, "gene_info.yml")
+SCHEMA_DEFINITIONS_DIR = os.path.join(PACKAGE_ROOT, "schema_definitions")
+SCHEMA_DEFINITION_FILE = os.path.join(SCHEMA_DEFINITIONS_DIR, "schema_definition.yaml")
+SCHEMA_REFERENCE_BASE_URL = "https://github.com/chanzuckerberg/single-cell-curation/blob/main/schema"
+SCHEMA_REFERENCE_FILE_NAME = "schema.md"

hca_schema_validator/_vendored/cellxgene_schema/gencode.py

@@ -0,0 +1,163 @@
+import enum
+import gzip
+import os
+from typing import Union
+
+from . import env
+
+
+class SupportedOrganisms(enum.Enum):
+    # Note that this is different from the valid organisms in `organism_ontology_term_id`
+    HOMO_SAPIENS = "NCBITaxon:9606"
+    MUS_MUSCULUS = "NCBITaxon:10090"
+    SARS_COV_2 = "NCBITaxon:2697049"
+    ERCC = "NCBITaxon:32630"
+    DROSOPHILA_MELANOGASTER = "NCBITaxon:7227"
+    DANIO_RERIO = "NCBITaxon:7955"
+    CAENORHABDITIS_ELEGANS = "NCBITaxon:6239"
+    MACACA_FASCICULARIS = "NCBITaxon:9541"
+    ORYCTOLAGUS_CUNICULUS = "NCBITaxon:9986"
+    CALLITHRIX_JACCHUS = "NCBITaxon:9483"
+    GORILLA_GORILLA = "NCBITaxon:9595"
+    MACACA_MULATTA = "NCBITaxon:9544"
+    PAN_TROGLODYTES = "NCBITaxon:9598"
+    SUS_SCROFA = "NCBITaxon:9823"
+    MICROCEBUS_MURINUS = "NCBITaxon:30608"
+    RATTUS_NORVEGICUS = "NCBITaxon:10116"
+
+
+EXEMPT_ORGANISMS = ["NCBITaxon:2697049", "NCBITaxon:32630"]
+
+
+def get_organism_from_feature_id(
+    feature_id: str,
+) -> Union[SupportedOrganisms, None]:
+    """
+    Determines organism based on which gene file the feature id was in
+
+    :param str feature_id: the feature id
+
+    :rtype Union[ontology.SypportedOrganisms, None]
+    :return: the organism the feature id is from
+    """
+
+    for organism in SupportedOrganisms:
+        gene_checker = get_gene_checker(organism)
+        if gene_checker.is_valid_id(feature_id):
+            return organism
+
+    return None
+
+
+class GeneChecker:
+    """Handles checking gene ids, retrieves symbols"""
+
+    GENE_FILES = {
+        SupportedOrganisms.HOMO_SAPIENS: os.path.join(env.GENCODE_DIR, "genes_homo_sapiens.csv.gz"),
+        SupportedOrganisms.MUS_MUSCULUS: os.path.join(env.GENCODE_DIR, "genes_mus_musculus.csv.gz"),
+        SupportedOrganisms.SARS_COV_2: os.path.join(env.GENCODE_DIR, "genes_sars_cov_2.csv.gz"),
+        SupportedOrganisms.ERCC: os.path.join(env.GENCODE_DIR, "genes_ercc.csv.gz"),
+        SupportedOrganisms.DROSOPHILA_MELANOGASTER: os.path.join(
+            env.GENCODE_DIR, "genes_drosophila_melanogaster.csv.gz"
+        ),
+        SupportedOrganisms.DANIO_RERIO: os.path.join(env.GENCODE_DIR, "genes_danio_rerio.csv.gz"),
+        SupportedOrganisms.CAENORHABDITIS_ELEGANS: os.path.join(env.GENCODE_DIR, "genes_caenorhabditis_elegans.csv.gz"),
+        SupportedOrganisms.MACACA_FASCICULARIS: os.path.join(env.GENCODE_DIR, "genes_macaca_fascicularis.csv.gz"),
+        SupportedOrganisms.ORYCTOLAGUS_CUNICULUS: os.path.join(env.GENCODE_DIR, "genes_oryctolagus_cuniculus.csv.gz"),
+        SupportedOrganisms.CALLITHRIX_JACCHUS: os.path.join(env.GENCODE_DIR, "genes_callithrix_jacchus.csv.gz"),
+        SupportedOrganisms.GORILLA_GORILLA: os.path.join(env.GENCODE_DIR, "genes_gorilla_gorilla.csv.gz"),
+        SupportedOrganisms.MACACA_MULATTA: os.path.join(env.GENCODE_DIR, "genes_macaca_mulatta.csv.gz"),
+        SupportedOrganisms.PAN_TROGLODYTES: os.path.join(env.GENCODE_DIR, "genes_pan_troglodytes.csv.gz"),
+        SupportedOrganisms.SUS_SCROFA: os.path.join(env.GENCODE_DIR, "genes_sus_scrofa.csv.gz"),
+        SupportedOrganisms.MICROCEBUS_MURINUS: os.path.join(env.GENCODE_DIR, "genes_microcebus_murinus.csv.gz"),
+        SupportedOrganisms.RATTUS_NORVEGICUS: os.path.join(env.GENCODE_DIR, "genes_rattus_norvegicus.csv.gz"),
+    }
+
+    def __init__(self, species: SupportedOrganisms):
+        """
+        :param enum.Enum.SupportedSpecies species: item from SupportedOrganisms
+        """
+
+        if species not in self.GENE_FILES:
+            raise ValueError(f"{species} not supported.")
+
+        self.species = species
+        self.gene_dict = {}
+        with gzip.open(self.GENE_FILES[species], "rt") as genes:
+            for gene in genes:
+                gene = gene.rstrip().split(",")  # type: ignore
+                gene_id = gene[0]
+                gene_label = gene[1]
+                gene_length = int(gene[3])
+                gene_type = gene[4]
+
+                self.gene_dict[gene_id] = (gene_label, gene_length, gene_type)
+
+    def is_valid_id(self, gene_id: str) -> bool:
+        """
+        Checks for validity of gene id
+
+        :param str gene_id: ENSEMBL gene id
+
+        :rtype bool
+        :return True if the gene_id is a valid ENSEMBL id, False otherwise
+        """
+
+        return gene_id in self.gene_dict
+
+    def get_symbol(self, gene_id: str) -> str:
+        """
+        Gets symbol associated to the ENSEBML id
+
+        :param str gene_id: ENSEMBL gene id
+
+        :rtype str
+        :return A gene symbol
+        """
+
+        if self.is_valid_id(gene_id):
+            return self.gene_dict[gene_id][0]
+        else:
+            raise ValueError(f"The id '{gene_id}' is not a valid ENSEMBL id for '{self.species}'")
+
+    def get_length(self, gene_id: str) -> int:
+        """
+        Gets feature length associated to the ENSEBML id
+
+        :param str gene_id: ENSEMBL gene id
+
+        :rtype int
+        :return A gene length
+        """
+
+        if self.is_valid_id(gene_id):
+            return self.gene_dict[gene_id][1]
+        else:
+            raise ValueError(f"The id '{gene_id}' is not a valid ENSEMBL id for '{self.species}'")
+
+    def get_type(self, gene_id: str) -> str:
+        """
+        Gets feature type associated to the ENSEBML id
+
+        :param str gene_id: ENSEMBL gene id
+
+        :rtype str
+        :return A feature type
+        """
+
+        if self.is_valid_id(gene_id):
+            return self.gene_dict[gene_id][2]
+        else:
+            raise ValueError(f"The id '{gene_id}' is not a valid ENSEMBL id for '{self.species}'")
+
+
+# cache the gene checkers
+_gene_checkers = {}
+
+
+def get_gene_checker(species: SupportedOrganisms) -> GeneChecker:
+    # Values will be instances of gencode.GeneChecker,
+    # keys will be one of gencode.SupportedOrganisms
+    if species not in _gene_checkers:
+        _gene_checkers[species] = GeneChecker(species)
+    return _gene_checkers[species]

hca_schema_validator/_vendored/cellxgene_schema/gencode_files/.gitignore

@@ -0,0 +1,3 @@
+*owl
+*gtf.gz
+

hca_schema_validator/_vendored/cellxgene_schema/gencode_files/drosophila_melanogaster_diff.txt

@@ -0,0 +1,42 @@
+FBgn0011655
+FBgn0086613
+FBgn0038893
+FBgn0086059
+FBgn0263118
+FBgn0086060
+FBgn0051320
+FBgn0039632
+FBgn0086052
+FBgn0086058
+FBgn0086073
+FBti0062685
+FBgn0086082
+FBgn0086076
+FBgn0086077
+FBgn0082953
+FBgn0086080
+FBgn0086081
+FBti0062686
+FBgn0086026
+FBgn0082993
+FBgn0086022
+FBgn0086029
+FBgn0025830
+FBgn0086030
+FBgn0027585
+FBgn0086047
+FBgn0086044
+FBgn0086045
+FBgn0086046
+FBgn0086043
+FBgn0029656
+FBgn0026749
+FBgn0082990
+FBgn0086038
+FBgn0086032
+FBgn0086034
+FBgn0086031
+FBgn0086036
+FBgn0086035
+FBgn0086040
+FBgn0086037

hca_schema_validator/_vendored/cellxgene_schema/gencode_files/gene_info.md

@@ -0,0 +1,29 @@
+This file describes the expected format of *[gene_info.yml](./gene_info.yml)*. It is used to track the
+current version of gene data to be download from various sources.
+
+### FORMAT
+
+Bellow is a description of the expected fields.
+
+| Field       | type   | required | default | description                                                                                                                        |
+|-------------|--------|----------|---------|------------------------------------------------------------------------------------------------------------------------------------|
+| {id}        | object | True     |         | Used to identify the dataset.                                                                                                      |
+| description | str    | True     |         | Can only contain characters valid in a file name.                                                                                  |
+| version     | str    | False    | None    | Use to track the current version of the data used. A version must be present to support auto updating.                             |
+| url         | str    | True     |         | The URL pointing at the data to download or a python formatting string that can be used to build the URL by adding in the version. |
+
+### example
+
+```
+human:
+  description: homo_sapiens
+  version: '38'
+  url: http://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_{version}/gencode.v{version}.primary_assembly.annotation.gtf.gz      
+```
+
+### Updating
+
+There are two way to update *[gene_info.yml](./gene_info.yml)*.
+The first approach is to run the recipe `make genes_update`. This is the automated approach and will pull the latest
+version from Gencode and update `gene_info.yml` to reflect those changes. The second approach is to update the fields
+`version` in *[gene_info.yml](./gene_info.yml)* to the desired version.

hca_schema_validator/_vendored/cellxgene_schema/gencode_files/gene_info.yml

@@ -0,0 +1,51 @@
+# Changes to this file will trigger a run of the gencode_process.yml GHA job, which will run the gene_processing.py script.
+ercc:
+  description: ercc
+  url: https://assets.thermofisher.com/TFS-Assets/LSG/manuals/cms_095047.txt
+human:
+  description: homo_sapiens
+  url: https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_{version}/gencode.v{version}.primary_assembly.annotation.gtf.gz
+  version: 48
+mouse:
+  description: mus_musculus
+  url: https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_{version}/gencode.v{version}.primary_assembly.annotation.gtf.gz
+  version: M37
+sars_cov_2:
+  description: sars_cov_2
+  url: ftp://ftp.ensemblgenomes.org/pub/viruses/gtf/sars_cov_2/Sars_cov_2.ASM985889v3.101.gtf.gz
+fruit_fly:
+  description: drosophila_melanogaster
+  url: https://ftp.ensembl.org/pub/release-114/gtf/drosophila_melanogaster/Drosophila_melanogaster.BDGP6.54.114.gtf.gz
+zebrafish:
+  description: danio_rerio
+  url: https://ftp.ensembl.org/pub/release-114/gtf/danio_rerio/Danio_rerio.GRCz11.114.gtf.gz
+roundworm:
+  description: caenorhabditis_elegans
+  url: https://ftp.ensembl.org/pub/release-114/gtf/caenorhabditis_elegans/Caenorhabditis_elegans.WBcel235.114.gtf.gz
+macaque:
+  description: macaca_fascicularis
+  url: https://ftp.ensembl.org/pub/release-114/gtf/macaca_fascicularis/Macaca_fascicularis.Macaca_fascicularis_6.0.114.gtf.gz
+rabbit:
+  description: oryctolagus_cuniculus
+  url: https://ftp.ensembl.org/pub/release-114/gtf/oryctolagus_cuniculus/Oryctolagus_cuniculus.OryCun2.0.114.gtf.gz
+marmoset:
+  description: callithrix_jacchus
+  url: https://ftp.ensembl.org/pub/release-114/gtf/callithrix_jacchus/Callithrix_jacchus.mCalJac1.pat.X.114.gtf.gz
+gorilla:
+  description: gorilla_gorilla
+  url: https://ftp.ensembl.org/pub/release-114/gtf/gorilla_gorilla/Gorilla_gorilla.gorGor4.114.gtf.gz
+rhesus_macaque:
+  description: macaca_mulatta
+  url: https://ftp.ensembl.org/pub/release-114/gtf/macaca_mulatta/Macaca_mulatta.Mmul_10.114.gtf.gz
+troglodyte:
+  description: pan_troglodytes
+  url: https://ftp.ensembl.org/pub/release-114/gtf/pan_troglodytes/Pan_troglodytes.Pan_tro_3.0.114.gtf.gz
+pig:
+  description: sus_scrofa
+  url: https://ftp.ensembl.org/pub/release-114/gtf/sus_scrofa/Sus_scrofa.Sscrofa11.1.114.gtf.gz
+lemur:
+  description: microcebus_murinus
+  url: https://ftp.ensembl.org/pub/release-114/gtf/microcebus_murinus/Microcebus_murinus.Mmur_3.0.114.gtf.gz
+rat: 
+  description: rattus_norvegicus
+  url: https://ftp.ensembl.org/pub/release-114/gtf/rattus_norvegicus/Rattus_norvegicus.GRCr8.114.gtf.gz