scdataloader 1.8.0__tar.gz → 1.9.0__tar.gz

{scdataloader-1.8.0 → scdataloader-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdataloader
-Version: 1.8.0
+Version: 1.9.0
 Summary: a dataloader for single cell data in lamindb
 Project-URL: repository, https://github.com/jkobject/scDataLoader
 Author-email: jkobject <jkobject@gmail.com>
@@ -14,13 +14,14 @@ Requires-Dist: cellxgene-census>=0.1.0
 Requires-Dist: django>=4.0.0
 Requires-Dist: harmonypy>=0.0.10
 Requires-Dist: ipykernel>=6.20.0
+Requires-Dist: jupytext>=1.16.0
 Requires-Dist: lamindb[bionty,cellregistry,jupyter,ourprojects,zarr]<2,>=1.0.4
 Requires-Dist: leidenalg>=0.8.0
-Requires-Dist: lightning>=2.0.0
 Requires-Dist: matplotlib>=3.5.0
 Requires-Dist: numpy==1.26.0
 Requires-Dist: palantir>=1.3.3
 Requires-Dist: pandas>=2.0.0
+Requires-Dist: pytorch-lightning>=2.3.0
 Requires-Dist: scikit-misc>=0.5.0
 Requires-Dist: seaborn>=0.11.0
 Requires-Dist: torch==2.2.0

{scdataloader-1.8.0 → scdataloader-1.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scdataloader"
-version = "1.8.0"
+version = "1.9.0"
 description = "a dataloader for single cell data in lamindb"
 authors = [
     {name = "jkobject", email = "jkobject@gmail.com"}
@@ -14,7 +14,7 @@ dependencies = [
     "lamindb[bionty,ourprojects,jupyter,cellregistry,zarr]>=1.0.4,<2",
     "cellxgene-census>=0.1.0",
     "torch==2.2.0",
-    "lightning>=2.0.0",
+    "pytorch-lightning>=2.3.0",
     "anndata>=0.9.0",
     "zarr>=2.10.0",
     "matplotlib>=3.5.0",
@@ -28,6 +28,8 @@ dependencies = [
     "scikit-misc>=0.5.0",
     "palantir>=1.3.3",
     "harmonypy>=0.0.10",
+    "jupytext>=1.16.0",
+
 ]
 
 [project.optional-dependencies]

scdataloader-1.9.0/scdataloader/VERSION

@@ -0,0 +1 @@
+1.9.0

{scdataloader-1.8.0 → scdataloader-1.9.0}/scdataloader/__init__.py

@@ -2,5 +2,6 @@ from .collator import Collator
 from .data import Dataset, SimpleAnnDataset
 from .datamodule import DataModule
 from .preprocess import Preprocessor
+from importlib.metadata import version
 
-__version__ = "1.7.0"
+__version__ = version("scdataloader")

{scdataloader-1.8.0 → scdataloader-1.9.0}/scdataloader/collator.py

@@ -24,7 +24,6 @@ class Collator:
         genelist: list[str] = [],
         downsample: Optional[float] = None,  # don't use it for training!
         save_output: Optional[str] = None,
-        metacell_mode: bool = False,
     ):
         """
         This class is responsible for collating data for the scPRINT model. It handles the
@@ -62,7 +61,6 @@ class Collator:
                 This is usually done by the scPRINT model during training but this option allows you to do it directly from the collator
             save_output (str, optional): If not None, saves the output to a file. Defaults to None.
                 This is mainly for debugging purposes
-            metacell_mode (bool, optional): Whether to sample a metacell. Defaults to False.
         """
         self.organisms = organisms
         self.genedf = load_genes(organisms)
@@ -82,7 +80,6 @@ class Collator:
         self.accepted_genes = {}
         self.downsample = downsample
         self.to_subset = {}
-        self.metacell_mode = metacell_mode
         self._setup(org_to_id, valid_genes, genelist)
 
     def _setup(self, org_to_id=None, valid_genes=[], genelist=[]):
@@ -135,6 +132,7 @@ class Collator:
         dataset = []
         nnz_loc = []
         is_meta = []
+        knn_cells = []
         for elem in batch:
             organism_id = elem[self.organism_name]
             if organism_id not in self.organism_ids:
@@ -145,10 +143,20 @@ class Collator:
             total_count.append(expr.sum())
             if len(self.accepted_genes) > 0:
                 expr = expr[self.accepted_genes[organism_id]]
+                if "knn_cells" in elem:
+                    elem["knn_cells"] = elem["knn_cells"][
+                        :, self.accepted_genes[organism_id]
+                    ]
             if self.how == "most expr":
                 nnz_loc = np.where(expr > 0)[0]
-                ma = self.max_len if self.max_len < len(nnz_loc) else len(nnz_loc)
-                loc = np.argsort(expr)[-(ma):][::-1]
+                if "knn_cells" in elem:
+                    nnz_loc = np.where(expr + elem["knn_cells"].sum(0) > 0)[0]
+                    ma = self.max_len if self.max_len < len(nnz_loc) else len(nnz_loc)
+                    loc = np.argsort(expr + elem["knn_cells"].mean(0))[-(ma):][::-1]
+                else:
+                    nnz_loc = np.where(expr > 0)[0]
+                    ma = self.max_len if self.max_len < len(nnz_loc) else len(nnz_loc)
+                    loc = np.argsort(expr)[-(ma):][::-1]
                 # nnz_loc = [1] * 30_000
                 # loc = np.argsort(expr)[-(self.max_len) :][::-1]
             elif self.how == "random expr":
@@ -171,33 +179,49 @@ class Collator:
                 "all",
                 "some",
             ]:
-                zero_loc = np.where(expr == 0)[0]
-                zero_loc = zero_loc[
-                    np.random.choice(
-                        len(zero_loc),
-                        self.add_zero_genes
-                        + (
-                            0
-                            if self.max_len < len(nnz_loc)
-                            else self.max_len - len(nnz_loc)
-                        ),
-                        replace=False,
-                    )
-                ]
+                if "knn_cells" in elem:
+                    # we complete with genes expressed in the knn
+                    nnz_loc = np.where(elem["knn_cells"].sum(0) > 0)[0]
+                    ma = self.max_len if self.max_len < len(nnz_loc) else len(nnz_loc)
+                    # which is not a zero_loc in this context
+                    zero_loc = np.argsort(elem["knn_cells"].sum(0))[-(ma):][::-1]
+                else:
+                    zero_loc = np.where(expr == 0)[0]
+                    zero_loc = zero_loc[
+                        np.random.choice(
+                            len(zero_loc),
+                            self.add_zero_genes
+                            + (
+                                0
+                                if self.max_len < len(nnz_loc)
+                                else self.max_len - len(nnz_loc)
+                            ),
+                            replace=False,
+                        )
+                    ]
                 loc = np.concatenate((loc, zero_loc), axis=None)
             expr = expr[loc]
-            loc = loc + self.start_idx[organism_id]
+            if "knn_cells" in elem:
+                elem["knn_cells"] = elem["knn_cells"][:, loc]
             if self.how == "some":
+                if "knn_cells" in elem:
+                    elem["knn_cells"] = elem["knn_cells"][
+                        :, self.to_subset[organism_id]
+                    ]
                 expr = expr[self.to_subset[organism_id]]
                 loc = loc[self.to_subset[organism_id]]
             exprs.append(expr)
-            gene_locs.append(loc)
+            if "knn_cells" in elem:
+                knn_cells.append(elem["knn_cells"])
+            # then we need to add the start_idx to the loc to give it the correct index
+            # according to the model
+            gene_locs.append(loc + self.start_idx[organism_id])
 
             if self.tp_name is not None:
                 tp.append(elem[self.tp_name])
             else:
                 tp.append(0)
-            if self.metacell_mode:
+            if "is_meta" in elem:
                 is_meta.append(elem["is_meta"])
             other_classes.append([elem[i] for i in self.class_names])
         expr = np.array(exprs)
@@ -207,6 +231,7 @@ class Collator:
         other_classes = np.array(other_classes)
         dataset = np.array(dataset)
         is_meta = np.array(is_meta)
+        knn_cells = np.array(knn_cells)
         # normalize counts
         if self.norm_to is not None:
             expr = (expr * self.norm_to) / total_count[:, None]
@@ -217,15 +242,6 @@ class Collator:
         if self.n_bins:
             pass
 
-        # find the associated gene ids (given the species)
-
-        # get the NN cells
-
-        # do encoding / selection a la scGPT
-
-        # do encoding of graph location
-        # encode all the edges in some sparse way
-        # normalizing total counts between 0,1
         ret = {
             "x": Tensor(expr),
             "genes": Tensor(gene_locs).int(),
@@ -233,8 +249,10 @@ class Collator:
             "tp": Tensor(tp),
             "depth": Tensor(total_count),
         }
-        if self.metacell_mode:
+        if len(is_meta) > 0:
             ret.update({"is_meta": Tensor(is_meta).int()})
+        if len(knn_cells) > 0:
+            ret.update({"knn_cells": Tensor(knn_cells)})
         if len(dataset) > 0:
             ret.update({"dataset": Tensor(dataset).to(long)})
         if self.downsample is not None:
@@ -242,6 +260,8 @@ class Collator:
         if self.save_output is not None:
             with open(self.save_output, "a") as f:
                 np.savetxt(f, ret["x"].numpy())
+            with open(self.save_output + "_loc", "a") as f:
+                np.savetxt(f, gene_locs)
         return ret
 
 

{scdataloader-1.8.0 → scdataloader-1.9.0}/scdataloader/config.py

@@ -118,7 +118,7 @@ MAIN_HUMAN_MOUSE_DEV_STAGE_MAP = {
     ],
     "HsapDv:0000258": [  # mature stage
         "MmusDv:0000110",  # mature stage
-        "HsapDv:0000204",
+        "HsapDv:0000204", # 
     ],
     "HsapDv:0000227": [  # late adult stage
         "MmusDv:0000091",  # 20 month-old stage

{scdataloader-1.8.0 → scdataloader-1.9.0}/scdataloader/data.py

@@ -58,6 +58,7 @@ class Dataset(torchDataset):
     hierarchical_clss: Optional[list[str]] = field(default_factory=list)
     join_vars: Literal["inner", "outer"] | None = None
     metacell_mode: float = 0.0
+    get_knn_cells: bool = False
 
     def __post_init__(self):
         self.mapped_dataset = mapped(
@@ -69,6 +70,7 @@ class Dataset(torchDataset):
             stream=True,
             parallel=True,
             metacell_mode=self.metacell_mode,
+            get_knn_cells=self.get_knn_cells,
         )
         print(
             "won't do any check but we recommend to have your dataset coming from local storage"
@@ -371,6 +373,7 @@ def mapped(
     is_run_input: bool | None = None,
     metacell_mode: bool = False,
     meta_assays: list[str] = ["EFO:0022857", "EFO:0010961"],
+    get_knn_cells: bool = False,
 ) -> MappedCollection:
     path_list = []
     for artifact in dataset.artifacts.all():
@@ -397,5 +400,6 @@ def mapped(
         dtype=dtype,
         meta_assays=meta_assays,
         metacell_mode=metacell_mode,
+        get_knn_cells=get_knn_cells,
     )
     return ds

{scdataloader-1.8.0 → scdataloader-1.9.0}/scdataloader/datamodule.py

@@ -52,6 +52,7 @@ class DataModule(L.LightningDataModule):
             # "EFO:0030062", # slide-seq
         ],
         metacell_mode: float = 0.0,
+        get_knn_cells: bool = False,
         modify_seed_on_requeue: bool = True,
         **kwargs,
     ):
@@ -88,6 +89,7 @@ class DataModule(L.LightningDataModule):
             metacell_mode (float, optional): The probability of using metacell mode. Defaults to 0.0.
             clss_to_predict (list, optional): List of classes to predict. Defaults to ["organism_ontology_term_id"].
             modify_seed_on_requeue (bool, optional): Whether to modify the seed on requeue. Defaults to True.
+            get_knn_cells (bool, optional): Whether to get the k-nearest neighbors of each queried cells. Defaults to False.
             **kwargs: Additional keyword arguments passed to the pytorch DataLoader.
             see @file data.py and @file collator.py for more details about some of the parameters
         """
@@ -98,6 +100,7 @@ class DataModule(L.LightningDataModule):
                 clss_to_predict=clss_to_predict,
                 hierarchical_clss=hierarchical_clss,
                 metacell_mode=metacell_mode,
+                get_knn_cells=get_knn_cells,
             )
         # and location
         self.metacell_mode = bool(metacell_mode)
@@ -157,7 +160,6 @@ class DataModule(L.LightningDataModule):
                 tp_name=tp_name,
                 organism_name=organism_name,
                 class_names=clss_to_predict,
-                metacell_mode=bool(metacell_mode),
             )
         self.validation_split = validation_split
         self.test_split = test_split

{scdataloader-1.8.0 → scdataloader-1.9.0}/scdataloader/mapped.py

@@ -96,8 +96,9 @@ class MappedCollection:
         cache_categories: Enable caching categories of ``obs_keys`` for faster access.
         parallel: Enable sampling with multiple processes.
         dtype: Convert numpy arrays from ``.X``, ``.layers`` and ``.obsm``
-        meta_assays: Assays to check for metacells.
-        metacell_mode: Mode for metacells.
+        meta_assays: Assays that are already defined as metacells.
+        metacell_mode: frequency at which to sample a metacell (an average of k-nearest neighbors).
+        get_knn_cells: Whether to also dataload the k-nearest neighbors of each queried cells.
     """
 
     def __init__(
@@ -114,6 +115,7 @@ class MappedCollection:
         parallel: bool = False,
         dtype: str | None = None,
         metacell_mode: float = 0.0,
+        get_knn_cells: bool = False,
         meta_assays: list[str] = ["EFO:0022857", "EFO:0010961"],
     ):
         if join not in {None, "inner", "outer"}:  # pragma: nocover
@@ -166,6 +168,7 @@ class MappedCollection:
         self.metacell_mode = metacell_mode
         self.path_list = path_list
         self.meta_assays = meta_assays
+        self.get_knn_cells = get_knn_cells
         self._make_connections(path_list, parallel)
 
         self._cache_cats: dict = {}
@@ -396,12 +399,15 @@ class MappedCollection:
                         label_idx = self.encoders[label][label_idx]
                     out[label] = label_idx
 
-            out["is_meta"] = False
-            if len(self.meta_assays) > 0 and "assay_ontology_term_id" in self.obs_keys:
-                if out["assay_ontology_term_id"] in self.meta_assays:
-                    out["is_meta"] = True
-                    return out
             if self.metacell_mode > 0:
+                if (
+                    len(self.meta_assays) > 0
+                    and "assay_ontology_term_id" in self.obs_keys
+                ):
+                    if out["assay_ontology_term_id"] in self.meta_assays:
+                        out["is_meta"] = True
+                        return out
+                out["is_meta"] = False
                 if np.random.random() < self.metacell_mode:
                     out["is_meta"] = True
                     distances = self._get_data_idx(store["obsp"]["distances"], obs_idx)
@@ -410,6 +416,19 @@ class MappedCollection:
                         out[layers_key] += self._get_data_idx(
                             lazy_data, i, self.join_vars, var_idxs_join, self.n_vars
                         )
+            elif self.get_knn_cells:
+                distances = self._get_data_idx(store["obsp"]["distances"], obs_idx)
+                nn_idx = np.argsort(-1 / (distances - 1e-6))[:6]
+                out["knn_cells"] = np.array(
+                    [
+                        self._get_data_idx(
+                            lazy_data, i, self.join_vars, var_idxs_join, self.n_vars
+                        )
+                        for i in nn_idx
+                    ],
+                    dtype=int,
+                )
+                out["distances"] = distances[nn_idx]
 
         return out
 

{scdataloader-1.8.0 → scdataloader-1.9.0}/scdataloader/preprocess.py

@@ -9,7 +9,7 @@ import scanpy as sc
 from anndata import AnnData, read_h5ad
 from scipy.sparse import csr_matrix
 from upath import UPath
-
+import gc
 from scdataloader import utils as data_utils
 
 FULL_LENGTH_ASSAYS = [
@@ -18,7 +18,7 @@ FULL_LENGTH_ASSAYS = [
     "EFO:0008931",
 ]
 
-MAXFILESIZE = 10_000_000_000
+MAXFILESIZE = 5_000_000_000
 
 
 class Preprocessor:
@@ -64,6 +64,11 @@ class Preprocessor:
         """
         Initializes the preprocessor and configures the workflow steps.
 
+        Your dataset should contain at least the following obs:
+        - `organism_ontology_term_id` with the ontology id of the organism of your anndata
+        - gene names in the `var.index` field of your anndata that map to the ensembl_gene nomenclature
+        or the hugo gene symbols nomenclature (if the later, set `is_symbol` to True)
+
         Args:
             filter_gene_by_counts (int or bool, optional): Determines whether to filter genes by counts.
                 If int, filters genes with counts. Defaults to False.
@@ -130,13 +135,21 @@ class Preprocessor:
         self.keepdata = keepdata
 
     def __call__(self, adata, dataset_id=None) -> AnnData:
-        if adata[0].obs.organism_ontology_term_id.iloc[0] not in self.organisms:
+        if self.additional_preprocess is not None:
+            adata = self.additional_preprocess(adata)
+        if "organism_ontology_term_id" not in adata[0].obs.columns:
+            raise ValueError(
+                "organism_ontology_term_id not found in adata.obs, you need to add an ontology term id for the organism of your anndata"
+            )
+        if not adata[0].var.index.str.contains("ENS").any() and not self.is_symbol:
+            raise ValueError(
+                "gene names in the `var.index` field of your anndata should map to the ensembl_gene nomenclature else set `is_symbol` to True if using hugo symbols"
+            )
+        if adata.obs["organism_ontology_term_id"].iloc[0] not in self.organisms:
             raise ValueError(
                 "we cannot work with this organism",
-                adata[0].obs.organism_ontology_term_id.iloc[0],
+                adata.obs["organism_ontology_term_id"],
             )
-        if self.additional_preprocess is not None:
-            adata = self.additional_preprocess(adata)
         if adata.raw is not None and self.use_raw:
             adata.X = adata.raw.X
             del adata.raw
@@ -152,11 +165,12 @@ class Preprocessor:
                 del adata.layers
         if len(adata.varm.keys()) > 0 and not self.keepdata:
             del adata.varm
-        if len(adata.obsm.keys()) > 0 and self.do_postp and not self.keepdata:
+        if len(adata.obsm.keys()) > 0 and not self.keepdata:
             del adata.obsm
-        if len(adata.obsp.keys()) > 0 and self.do_postp and not self.keepdata:
+        if len(adata.obsp.keys()) > 0 and not self.keepdata:
             del adata.obsp
         # check that it is a count
+
         print("checking raw counts")
         if np.abs(
             adata[:50_000].X.astype(int) - adata[:50_000].X
@@ -217,23 +231,51 @@ class Preprocessor:
             )
         )
 
-        if self.is_symbol or not adata.var.index.str.contains("ENS").any():
-            if not adata.var.index.str.contains("ENS").any():
-                print("No ENS genes found, assuming gene symbols...")
-            genesdf["ensembl_gene_id"] = genesdf.index
-            var = (
-                adata.var.merge(
-                    genesdf.drop_duplicates("symbol").set_index("symbol", drop=False),
-                    left_index=True,
-                    right_index=True,
-                    how="inner",
-                )
-                .sort_values(by="ensembl_gene_id")
-                .set_index("ensembl_gene_id")
+        # Check if we have a mix of gene names and ensembl IDs
+        has_ens = adata.var.index.str.match(r"ENS.*\d{6,}$").any()
+        all_ens = adata.var.index.str.match(r"ENS.*\d{6,}$").all()
+
+        if not has_ens:
+            print("No ENS genes found, assuming gene symbols...")
+        elif not all_ens:
+            print("Mix of ENS and gene symbols found, converting all to ENS IDs...")
+
+        genesdf["ensembl_gene_id"] = genesdf.index
+
+        # For genes that are already ENS IDs, use them directly
+        ens_mask = adata.var.index.str.match(r"ENS.*\d{6,}$")
+        symbol_mask = ~ens_mask
+
+        # Handle symbol genes
+        if symbol_mask.any():
+            symbol_var = adata.var[symbol_mask].merge(
+                genesdf.drop_duplicates("symbol").set_index("symbol", drop=False),
+                left_index=True,
+                right_index=True,
+                how="inner",
+            )
+
+        # Handle ENS genes
+        if ens_mask.any():
+            ens_var = adata.var[ens_mask].merge(
+                genesdf, left_index=True, right_index=True, how="inner"
             )
-            adata = adata[:, var["symbol"]]
-            adata.var = var
-            genesdf = genesdf.set_index("ensembl_gene_id")
+
+        # Combine and sort
+        if symbol_mask.any() and ens_mask.any():
+            var = pd.concat([symbol_var, ens_var])
+        elif symbol_mask.any():
+            var = symbol_var
+        else:
+            var = ens_var
+
+        adata = adata[:, var.index]
+        var = var.sort_values(by="ensembl_gene_id").set_index("ensembl_gene_id")
+        # Update adata with combined genes
+        adata.var = var
+        genesdf = genesdf.set_index("ensembl_gene_id")
+        # Drop duplicate genes, keeping first occurrence
+        adata = adata[:, ~adata.var.index.duplicated(keep="first")]
 
         intersect_genes = set(adata.var.index).intersection(set(genesdf.index))
         print(f"Removed {len(adata.var.index) - len(intersect_genes)} genes.")
@@ -462,13 +504,17 @@ class LaminPreprocessor(Preprocessor):
                 print(file)
 
                 path = cache_path(file) if self.force_preloaded else file.cache()
-                backed = read_h5ad(path, backed="r")
-                if backed.obs.is_primary_data.sum() == 0:
-                    print(f"{file.key} only contains non primary cells.. dropping")
-                    # Save the stem_uid to a file to avoid loading it again
+                backed = file.open()
+                # backed = read_h5ad(path, backed="r")
+                if "is_primary_data" in backed.obs.columns:
+                    if backed.obs.is_primary_data.sum() == 0:
+                        print(f"{file.key} only contains non primary cells.. dropping")
+                        # Save the stem_uid to a file to avoid loading it again
                     with open("nonprimary.txt", "a") as f:
                         f.write(f"{file.stem_uid}\n")
                     continue
+                else:
+                    print("Warning: couldn't check unicity from is_primary_data column")
                 if backed.shape[1] < 1000:
                     print(
                         f"{file.key} only contains less than 1000 genes and is likely not scRNAseq... dropping"
@@ -489,16 +535,23 @@ class LaminPreprocessor(Preprocessor):
                         block_size = int(
                             (np.ceil(badata.shape[0] / 30_000) * 30_000) // num_blocks
                         )
-                        print("num blocks ", num_blocks)
+                        print(
+                            "num blocks ",
+                            num_blocks,
+                            "block size ",
+                            block_size,
+                            "total elements ",
+                            badata.shape[0],
+                        )
                         for j in range(num_blocks):
-                            if j == 0 and i == 390:
-                                continue
                             start_index = j * block_size
                             end_index = min((j + 1) * block_size, badata.shape[0])
-                            block = badata[start_index:end_index].to_memory()
+                            block = badata[start_index:end_index]
+                            block = block.to_memory()
                             print(block)
                             block = super().__call__(
-                                block, dataset_id=file.stem_uid + "_p" + str(j)
+                                block,
+                                dataset_id=file.stem_uid + "_p" + str(j),
                             )
                             myfile = ln.Artifact.from_anndata(
                                 block,
@@ -508,16 +561,19 @@ class LaminPreprocessor(Preprocessor):
                                 + " p"
                                 + str(j)
                                 + " ( revises file "
-                                + str(file.key)
+                                + str(file.stem_uid)
                                 + " )",
                                 version=version,
                             )
                             myfile.save()
+
                             if self.keep_files:
                                 files.append(myfile)
+                                del block
                             else:
                                 del myfile
                                 del block
+                            gc.collect()
 
                     else:
                         adata = super().__call__(adata, dataset_id=file.stem_uid)
@@ -530,6 +586,7 @@ class LaminPreprocessor(Preprocessor):
                         myfile.save()
                         if self.keep_files:
                             files.append(myfile)
+                            del adata
                         else:
                             del myfile
                             del adata
@@ -549,7 +606,12 @@ class LaminPreprocessor(Preprocessor):
 
                 # issues with KLlggfw6I6lvmbqiZm46
             if self.keep_files:
-                dataset = ln.Collection(files, name=name, description=description)
+                # Reconstruct collection using keys
+                dataset = ln.Collection(
+                    [ln.Artifact.filter(key=k).one() for k in files],
+                    name=name,
+                    description=description,
+                )
                 dataset.save()
                 return dataset
             else:

{scdataloader-1.8.0 → scdataloader-1.9.0}/scdataloader/utils.py

@@ -154,7 +154,7 @@ def getBiomartTable(
     return res
 
 
-def validate(adata: AnnData, organism: str, need_all=True):
+def validate(adata: AnnData, organism: str, need_all=False):
     """
     validate checks if the adata object is valid for lamindb
 
@@ -578,7 +578,6 @@ def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10
 
 
 def populate_my_ontology(
-    organisms: List[str] = ["NCBITaxon:10090", "NCBITaxon:9606"],
     sex: List[str] = ["PATO:0000384", "PATO:0000383"],
     celltypes: List[str] = [],
     ethnicities: List[str] = [],
@@ -586,7 +585,7 @@ def populate_my_ontology(
     tissues: List[str] = [],
     diseases: List[str] = [],
     dev_stages: List[str] = [],
-    organism_clade: str = "vertebrates",
+    organisms_clade: List[str] = ["vertebrates", "plants"],
 ):
     """
     creates a local version of the lamin ontologies and add the required missing values in base ontologies
@@ -622,23 +621,27 @@ def populate_my_ontology(
             ln.save(records)
         bt.CellType(name="unknown", ontology_id="unknown").save()
     # Organism
-    if organisms is not None:
-        names = (
-            bt.Organism.public(organism=organism_clade).df().index
-            if not organisms
-            else organisms
-        )
-        source = bt.PublicSource.filter(name="ensembl", organism=organism_clade).last()
-        records = [
-            organism_or_organismlist
-            if isinstance(organism_or_organismlist, bt.Organism)
-            else organism_or_organismlist[0]
-            for organism_or_organismlist in [
-                bt.Organism.from_source(ontology_id=name, source=source)
-                for name in names
+    if organisms_clade is not None:
+        records = []
+        for organism_clade in organisms_clade:
+            names = bt.Organism.public(organism=organism_clade).df().index
+            source = bt.PublicSource.filter(
+                name="ensembl", organism=organism_clade
+            ).last()
+            records += [
+                bt.Organism.from_source(name=name, source=source) for name in names
             ]
-        ]
-        ln.save(records)
+        nrecords = []
+        prevrec = set()
+        for rec in records:
+            if rec is None:
+                continue
+            if not isinstance(rec, bt.Organism):
+                rec = rec[0]
+            if rec.uid not in prevrec:
+                nrecords.append(rec)
+                prevrec.add(rec.uid)
+        ln.save(nrecords)
         bt.Organism(name="unknown", ontology_id="unknown").save()
     # Phenotype
     if sex is not None:

scdataloader-1.8.0/scdataloader/VERSION

@@ -1 +0,0 @@
-1.8.0