scdataloader 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

scdataloader/preprocess.py

@@ -1,6 +1,6 @@
 import gc
 import time
-from typing import Callable, Optional, Union
+from typing import Callable, List, Optional, Union
 from uuid import uuid4
 
 import anndata as ad
@@ -16,7 +16,7 @@ from upath import UPath
 from scdataloader import utils as data_utils
 
 FULL_LENGTH_ASSAYS = [
-    "EFO: 0700016",
+    "EFO:0700016",
     "EFO:0008930",
     "EFO:0008931",
 ]
@@ -49,18 +49,18 @@ class Preprocessor:
         maxdropamount: int = 50,
         madoutlier: int = 5,
         pct_mt_outlier: int = 8,
-        batch_keys: list[str] = [
+        batch_keys: List[str] = [
             "assay_ontology_term_id",
             "self_reported_ethnicity_ontology_term_id",
             "sex_ontology_term_id",
             "donor_id",
             "suspension_type",
         ],
-        skip_validate: bool = False,
+        skip_validate: bool = True,
         additional_preprocess: Optional[Callable[[AnnData], AnnData]] = None,
         additional_postprocess: Optional[Callable[[AnnData], AnnData]] = None,
         do_postp: bool = True,
-        organisms: list[str] = ["NCBITaxon:9606", "NCBITaxon:10090"],
+        organisms: List[str] = ["NCBITaxon:9606", "NCBITaxon:10090"],
         use_raw: bool = True,
         keepdata: bool = False,
         drop_non_primary: bool = False,
@@ -145,9 +145,14 @@ class Preprocessor:
         if self.additional_preprocess is not None:
             adata = self.additional_preprocess(adata)
         if "organism_ontology_term_id" not in adata[0].obs.columns:
-            raise ValueError(
-                "organism_ontology_term_id not found in adata.obs, you need to add an ontology term id for the organism of your anndata"
-            )
+            if "organism_ontology_term_id" in adata.uns:
+                adata.obs["organism_ontology_term_id"] = adata.uns[
+                    "organism_ontology_term_id"
+                ]
+            else:
+                raise ValueError(
+                    "organism_ontology_term_id not found in adata.obs, you need to add an ontology term id for the organism of your anndata"
+                )
         if adata.obs["organism_ontology_term_id"].iloc[0] not in self.organisms:
             raise ValueError(
                 "we cannot work with this organism",
@@ -357,6 +362,8 @@ class Preprocessor:
             adata.obs["batches"] = adata.obs[batches].apply(
                 lambda x: ",".join(x.dropna().astype(str)), axis=1
             )
+            if "highly_variable" in adata.var.columns:
+                adata.var = adata.var.drop(columns=["highly_variable"])
             if self.n_hvg_for_postp:
                 try:
                     sc.pp.highly_variable_genes(
@@ -380,10 +387,13 @@ class Preprocessor:
                     )
             print("starting PCA")
             adata.obsm["X_pca"] = sc.pp.pca(
-                adata.layers["norm"][:, adata.var.highly_variable]
-                if "highly_variable" in adata.var.columns
-                else adata.layers["norm"],
-                n_comps=200 if adata.shape[0] > 200 else adata.shape[0] - 2,
+                (
+                    adata.layers["norm"][:, adata.var["highly_variable"]]
+                    if "highly_variable" in adata.var.columns
+                    else adata.layers["norm"]
+                ),
+                n_comps=50 if adata.shape[0] > 1000 else adata.shape[0] // 20,
+                chunked=adata.shape[0] > 100_000,
             )
 
             # additional
@@ -448,12 +458,14 @@ class LaminPreprocessor(Preprocessor):
         cache: bool = True,
         keep_files: bool = True,
         force_lamin_cache: bool = False,
+        assays_to_drop: List[str] = ["EFO:0008939"],
         **kwargs,
     ):
         super().__init__(*args, **kwargs)
         self.cache = cache
         self.keep_files = keep_files
         self.force_lamin_cache = force_lamin_cache
+        self.assays_to_drop = assays_to_drop
 
     def __call__(
         self,
@@ -499,11 +511,14 @@ class LaminPreprocessor(Preprocessor):
                     if backed.obs.is_primary_data.sum() == 0:
                         print(f"{file.key} only contains non primary cells.. dropping")
                         # Save the stem_uid to a file to avoid loading it again
-                    with open("nonprimary.txt", "a") as f:
-                        f.write(f"{file.stem_uid}\n")
-                    continue
+                        with open("nonprimary.txt", "a") as f:
+                            f.write(f"{file.stem_uid}\n")
+                        continue
                 else:
                     print("Warning: couldn't check unicity from is_primary_data column")
+                if backed.obs.assay_ontology_term_id[0] in self.assays_to_drop:
+                    print(f"{file.key} is in the assay drop list.. dropping")
+                    continue
                 if backed.shape[1] < 1000:
                     print(
                         f"{file.key} only contains less than 1000 genes and is likely not scRNAseq... dropping"
@@ -577,7 +592,7 @@ class LaminPreprocessor(Preprocessor):
                             try:
                                 myfile = ln.Artifact.from_anndata(
                                     adata,
-                                    revises=file,
+                                    # revises=file,
                                     description=description + " p" + str(i),
                                     version=version,
                                 )

scdataloader/utils.py

@@ -1,5 +1,7 @@
 import io
 import os
+import random
+import string
 import urllib
 from collections import Counter
 from functools import lru_cache
@@ -13,6 +15,7 @@ import torch
 from anndata import AnnData
 from biomart import BiomartServer
 from django.db import IntegrityError
+from lamindb.errors import DoesNotExist
 from scipy.sparse import csr_matrix
 from scipy.stats import median_abs_deviation
 from torch import Tensor
@@ -32,7 +35,9 @@ def fileToList(filename: str, strconv: callable = lambda x: x) -> list:
         return [strconv(val[:-1]) for val in f.readlines()]
 
 
-def listToFile(li: list, filename: str, strconv: callable = lambda x: str(x)) -> None:
+def listToFile(
+    li: List[str], filename: str, strconv: callable = lambda x: str(x)
+) -> None:
     """
     listToFile loads a list with [a,b,..] into an input file a\\n b\\n..
 
@@ -56,6 +61,23 @@ def slurm_restart_count(use_mine: bool = False):
         return int(os.getenv("SLURM_RESTART_COUNT", 0))
 
 
+def revert_to_raw(adata, mode="logp1"):
+    res = adata.X
+    if mode == "rlogp1":
+        res = np.exp(res) - 1
+    elif mode == "logp1":
+        res = (2**res) - 1
+    elif mode == "sqrt":
+        res = (res**2) - 1
+    res = (
+        (res.T / np.array([res[i][res[i] != 0].min() for i in range(res.shape[0])]))
+        .round()
+        .T
+    )  # .sum()
+    adata.X = res
+    return adata
+
+
 def createFoldersFor(filepath: str):
     """
     will recursively create folders if needed until having all the folders required to save the file in this filepath
@@ -68,7 +90,7 @@ def createFoldersFor(filepath: str):
 
 
 def _fetchFromServer(
-    ensemble_server: str, attributes: list, database: str = "hsapiens_gene_ensembl"
+    ensemble_server: str, attributes: List[str], database: str = "hsapiens_gene_ensembl"
 ):
     """
     Fetches data from the specified ensemble server.
@@ -274,7 +296,7 @@ def get_descendants(val, df):
     return r_onto | ontos
 
 
-def get_ancestry_mapping(all_elem: list, onto_df: pd.DataFrame):
+def get_ancestry_mapping(all_elem: List[str], onto_df: pd.DataFrame):
     """
     This function generates a mapping of all elements to their ancestors in the ontology dataframe.
 
@@ -370,12 +392,14 @@ def load_dataset_local(
     return dataset
 
 
-def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10090",
+def load_genes(
+    organisms: Union[str, List[str]] = "NCBITaxon:9606",
+):  # "NCBITaxon:10090",
     """
     Loads genes from the given organisms.
 
     Args:
-        organisms (Union[str, list]): The organisms to load genes from.
+        organisms (Union[str, List[str]]): The organisms to load genes from.
 
     Returns:
         pd.DataFrame: The genes dataframe.
@@ -583,6 +607,54 @@ def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10
     return organismdf
 
 
+def _adding_scbasecamp_genes(
+    species=[],
+):
+    if len(species) == 0:
+        species = set(
+            bt.Organism.using("laminlabs/arc-virtual-cell-atlas").df().ontology_id
+        )
+        -set(["NCBITaxon:10090", "NCBITaxon:9606"])
+    species = list(species)
+    if len(bt.Organism.filter(ontology_id="NCBITaxon:9593")) == 0:
+        bt.Organism(
+            name="gorilla gorilla",
+            ontology_id="NCBITaxon:9593",
+            scientific_name="Gorilla gorilla gorilla",
+        ).save()
+    if len(bt.Organism.filter(ontology_id="NCBITaxon:9594")) == 0:
+        bt.Organism(
+            name="rice",
+            ontology_id="NCBITaxon:4530",
+            scientific_name="Oryza sativa (main)",
+        ).save()
+
+    for i in species:
+        print(i)
+        df = (
+            bt.Gene.using("laminlabs/arc-virtual-cell-atlas")
+            .filter(organism__ontology_id=i)
+            .all()
+            .df()
+        )
+        genes = []
+        org = bt.Organism.filter(ontology_id=i).one()
+        ido = org.id
+        for row in df.to_dict(orient="records"):
+            row["organism_id"] = ido
+            gene = bt.Gene(
+                ensembl_gene_id=row["ensembl_gene_id"],
+                stable_id=row["stable_id"],
+                description=row["description"],
+                symbol=row["symbol"],
+                biotype=row["biotype"],
+                organism=org,
+                _skip_validation=True,
+            )
+            genes.append(gene)
+        ln.save(genes, ignore_conflicts=True)
+
+
 def populate_my_ontology(
     sex: List[str] = ["PATO:0000384", "PATO:0000383"],
     celltypes: List[str] = [],
@@ -591,7 +663,8 @@ def populate_my_ontology(
     tissues: List[str] = [],
     diseases: List[str] = [],
     dev_stages: List[str] = [],
-    organisms_clade: List[str] = ["vertebrates", "plants"],
+    organisms_clade: List[str] = ["vertebrates", "plants", "metazoa"],
+    genes_from: List[str] = ["NCBITaxon:10090", "NCBITaxon:9606"],
 ):
     """
     creates a local version of the lamin ontologies and add the required missing values in base ontologies
@@ -607,8 +680,6 @@ def populate_my_ontology(
     `df["assay_ontology_term_id"].unique()`
 
     Args:
-        lb (lamindb): lamindb instance.
-        organisms (list, optional): List of organisms. Defaults to ["NCBITaxon:10090", "NCBITaxon:9606"].
         sex (list, optional): List of sexes. Defaults to ["PATO:0000384", "PATO:0000383"].
         celltypes (list, optional): List of cell types. Defaults to [].
         ethnicities (list, optional): List of ethnicities. Defaults to [].
@@ -616,6 +687,7 @@ def populate_my_ontology(
         tissues (list, optional): List of tissues. Defaults to [].
         diseases (list, optional): List of diseases. Defaults to [].
         dev_stages (list, optional): List of developmental stages. Defaults to [].
+        organisms_clade (list, optional): List of organisms clade. Defaults to ["vertebrates", "plants"].
     """
     # cell type
     if celltypes is not None:
@@ -626,17 +698,17 @@ def populate_my_ontology(
             records = bt.CellType.from_values(names, field="ontology_id")
             ln.save(records)
         bt.CellType(name="unknown", ontology_id="unknown").save()
-    # Organism
+    # OrganismClade
     if organisms_clade is not None:
         records = []
         for organism_clade in organisms_clade:
             names = bt.Organism.public(organism=organism_clade).df().index
-            source = bt.PublicSource.filter(
-                name="ensembl", organism=organism_clade
-            ).last()
-            records += [
-                bt.Organism.from_source(name=name, source=source) for name in names
-            ]
+            source = bt.Source.filter(name="ensembl", organism=organism_clade).last()
+            for name in names:
+                try:
+                    records.append(bt.Organism.from_source(name=name, source=source))
+                except DoesNotExist:
+                    print(f"Organism {name} not found in source {source}")
         nrecords = []
         prevrec = set()
         for rec in records:
@@ -652,7 +724,7 @@ def populate_my_ontology(
     # Phenotype
     if sex is not None:
         names = bt.Phenotype.public().df().index if not sex else sex
-        source = bt.PublicSource.filter(name="pato").first()
+        source = bt.Source.filter(name="pato").first()
         records = [
             bt.Phenotype.from_source(ontology_id=i, source=source) for i in names
         ]
@@ -693,7 +765,7 @@ def populate_my_ontology(
     if dev_stages is not None:
         if len(dev_stages) == 0:
             bt.DevelopmentalStage.import_source()
-            source = bt.PublicSource.filter(organism="mouse", name="mmusdv").last()
+            source = bt.Source.filter(organism="mouse", name="mmusdv").last()
             bt.DevelopmentalStage.import_source(source=source)
         else:
             names = (
@@ -716,7 +788,7 @@ def populate_my_ontology(
         bt.Disease(name="normal", ontology_id="PATO:0000461").save()
         bt.Disease(name="unknown", ontology_id="unknown").save()
     # genes
-    for organism in ["NCBITaxon:10090", "NCBITaxon:9606"]:
+    for organism in genes_from:
         # convert onto to name
         organism = bt.Organism.filter(ontology_id=organism).one().name
         names = bt.Gene.public(organism=organism).df()["ensembl_gene_id"]
@@ -733,6 +805,29 @@ def populate_my_ontology(
             ln.save(records)
 
 
+def random_str(stringLength=6, stype="all", withdigits=True):
+    """
+    Generate a random string of letters and digits
+
+    Args:
+        stringLength (int, optional): the amount of char. Defaults to 6.
+        stype (str, optional): one of lowercase, uppercase, all. Defaults to 'all'.
+        withdigits (bool, optional): digits allowed in the string? Defaults to True.
+
+        Returns:
+        str: random string
+    """
+    if stype == "lowercase":
+        lettersAndDigits = string.ascii_lowercase
+    elif stype == "uppercase":
+        lettersAndDigits = string.ascii_uppercase
+    else:
+        lettersAndDigits = string.ascii_letters
+    if withdigits:
+        lettersAndDigits += string.digits
+    return "".join(random.choice(lettersAndDigits) for i in range(stringLength))
+
+
 def is_outlier(adata: AnnData, metric: str, nmads: int):
     """
     is_outlier detects outliers in adata.obs[metric]
@@ -798,11 +893,16 @@ def translate(
         obj = bt.Disease
     elif t == "self_reported_ethnicity_ontology_term_id":
         obj = bt.Ethnicity
+    elif t == "organism_ontology_term_id":
+        obj = bt.Organism
     else:
         return None
     if type(val) is str:
         return {val: obj.filter(ontology_id=val).one().name}
-    elif type(val) is list or type(val) is set:
-        return {i: obj.filter(ontology_id=i).one().name for i in set(val)}
     elif type(val) is dict or type(val) is Counter:
         return {obj.filter(ontology_id=k).one().name: v for k, v in val.items()}
+    elif type(val) is set:
+        return {i: obj.filter(ontology_id=i).one().name for i in val}
+    else:
+        rl = {i: obj.filter(ontology_id=i).one().name for i in set(val)}
+        return [rl.get(i, None) for i in val]

{scdataloader-2.0.0.dist-info → scdataloader-2.0.2.dist-info}/METADATA

@@ -1,29 +1,29 @@
 Metadata-Version: 2.4
 Name: scdataloader
-Version: 2.0.0
+Version: 2.0.2
 Summary: a dataloader for single cell data in lamindb
 Project-URL: repository, https://github.com/jkobject/scDataLoader
 Author-email: jkobject <jkobject@gmail.com>
 License-Expression: MIT
 License-File: LICENSE
 Keywords: dataloader,lamindb,pytorch,scPRINT,scRNAseq
-Requires-Python: <3.14,>=3.10
+Requires-Python: <3.13,>=3.10
 Requires-Dist: anndata>=0.9.0
 Requires-Dist: biomart>=0.9.0
 Requires-Dist: cellxgene-census>=0.1.0
 Requires-Dist: django>=4.0.0
 Requires-Dist: ipykernel>=6.20.0
 Requires-Dist: jupytext>=1.16.0
-Requires-Dist: lamindb[bionty,cellregistry,jupyter,zarr]==1.0.4
+Requires-Dist: lamindb[bionty,jupyter,zarr]==1.6.2
 Requires-Dist: leidenalg>=0.8.0
 Requires-Dist: lightning>=2.3.0
 Requires-Dist: matplotlib>=3.5.0
-Requires-Dist: numpy==1.26.0
+Requires-Dist: numpy<=2.2.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pytorch-lightning>=2.3.0
 Requires-Dist: scikit-misc>=0.5.0
 Requires-Dist: seaborn>=0.11.0
-Requires-Dist: torch==2.2.0
+Requires-Dist: torch>=2.2.0
 Requires-Dist: torchdata>=0.5.0
 Requires-Dist: zarr>=2.10.0
 Provides-Extra: dev

scdataloader-2.0.2.dist-info/RECORD

@@ -0,0 +1,16 @@
+scdataloader/__init__.py,sha256=Z5HURehoWw1GrecImmTXIkv4ih8Q5RxNQWPm8zjjXOA,226
+scdataloader/__main__.py,sha256=xPOtrEpQQQZUGTnm8KTvsQcA_jR45oMG_VHqd0Ny7_M,8677
+scdataloader/base.py,sha256=M1gD59OffRdLOgS1vHKygOomUoAMuzjpRtAfM3SBKF8,338
+scdataloader/collator.py,sha256=pITHfsWUkrUW7lMfgXfs1AfekgcfW9XfGHwi9LlKwm8,13651
+scdataloader/config.py,sha256=nM8J11z2-lornryy1KxDE9675Rcxge4RGhdmpeiMhuI,7173
+scdataloader/data.json,sha256=Zb8c27yk3rwMgtAU8kkiWWAyUwYBrlCqKUyEtaAx9i8,8785
+scdataloader/data.py,sha256=aiSpw4rd5L162ox2kuD-8ujWNix5fvVlXozdlfthMNU,18176
+scdataloader/datamodule.py,sha256=pGPPuxDrWz0GPBUz_vb4FUprbuNKkjq1hjr46m-fRVU,35783
+scdataloader/mapped.py,sha256=h9YKQ8SG9tyZL8c6_Wu5Xov5ODGK6FzVuFopz58xwN4,29887
+scdataloader/preprocess.py,sha256=4iqHqeSVE-oKRvwD0KKl_QH6HQWVYSMRPo9QNSq-3Pk,39179
+scdataloader/utils.py,sha256=Z6td0cIphrYDLVrPrV8q4jUC_HtwGQmi-NcbpdbWrns,31034
+scdataloader-2.0.2.dist-info/METADATA,sha256=QsyNBOyn_U9_TjFVbN-5WIGIkmKWf2sSksZUzKcNlqE,10314
+scdataloader-2.0.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+scdataloader-2.0.2.dist-info/entry_points.txt,sha256=VXAN1m_CjbdLJ6SKYR0sBLGDV4wvv31ri7fWWuwbpno,60
+scdataloader-2.0.2.dist-info/licenses/LICENSE,sha256=rGy_eYmnxtbOvKs7qt5V0czSWxJwgX_MlgMyTZwDHbc,1073
+scdataloader-2.0.2.dist-info/RECORD,,

{scdataloader-2.0.0.dist-info → scdataloader-2.0.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

scdataloader-2.0.0.dist-info/RECORD

@@ -1,16 +0,0 @@
-scdataloader/__init__.py,sha256=Z5HURehoWw1GrecImmTXIkv4ih8Q5RxNQWPm8zjjXOA,226
-scdataloader/__main__.py,sha256=3aZnqYrH8XDT9nW9Dbb3o9kr-sx1STmXDQHxBo_h_q0,8719
-scdataloader/base.py,sha256=M1gD59OffRdLOgS1vHKygOomUoAMuzjpRtAfM3SBKF8,338
-scdataloader/collator.py,sha256=pC5PVvxwyE7e84cdr5YC4ae85NbJubk6bfldIfOLFNE,12039
-scdataloader/config.py,sha256=nM8J11z2-lornryy1KxDE9675Rcxge4RGhdmpeiMhuI,7173
-scdataloader/data.json,sha256=Zb8c27yk3rwMgtAU8kkiWWAyUwYBrlCqKUyEtaAx9i8,8785
-scdataloader/data.py,sha256=skUBLX07cxC9gl4X0zDQ3rUpm0EJoUjngvdhL4lo5IA,18633
-scdataloader/datamodule.py,sha256=fKYjQlli9wm3uCJbD6FedooNSorjl5vfg5LuwDCQkQU,34388
-scdataloader/mapped.py,sha256=0erd1vCfCnUkdMuFO4Md0d78gNeNaHSWwRVXN4zhtQQ,29802
-scdataloader/preprocess.py,sha256=CA0pvfcfZ_nMhso7vl9MfKhKfaJbYwANqHfsa5Vyco4,38425
-scdataloader/utils.py,sha256=YzTCV1IkfXIaQmtdTXJvo_Vj1l_Dhau7UUM_BBccvL0,27939
-scdataloader-2.0.0.dist-info/METADATA,sha256=rNBXB06aOnCNTu8LjijBTReTototB2tRnGmnHaFtyVE,10328
-scdataloader-2.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-scdataloader-2.0.0.dist-info/entry_points.txt,sha256=VXAN1m_CjbdLJ6SKYR0sBLGDV4wvv31ri7fWWuwbpno,60
-scdataloader-2.0.0.dist-info/licenses/LICENSE,sha256=rGy_eYmnxtbOvKs7qt5V0czSWxJwgX_MlgMyTZwDHbc,1073
-scdataloader-2.0.0.dist-info/RECORD,,