scdataloader 1.8.1__tar.gz → 1.9.0__tar.gz

{scdataloader-1.8.1 → scdataloader-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdataloader
-Version: 1.8.1
+Version: 1.9.0
 Summary: a dataloader for single cell data in lamindb
 Project-URL: repository, https://github.com/jkobject/scDataLoader
 Author-email: jkobject <jkobject@gmail.com>
@@ -14,13 +14,14 @@ Requires-Dist: cellxgene-census>=0.1.0
 Requires-Dist: django>=4.0.0
 Requires-Dist: harmonypy>=0.0.10
 Requires-Dist: ipykernel>=6.20.0
+Requires-Dist: jupytext>=1.16.0
 Requires-Dist: lamindb[bionty,cellregistry,jupyter,ourprojects,zarr]<2,>=1.0.4
 Requires-Dist: leidenalg>=0.8.0
-Requires-Dist: lightning>=2.3.0
 Requires-Dist: matplotlib>=3.5.0
 Requires-Dist: numpy==1.26.0
 Requires-Dist: palantir>=1.3.3
 Requires-Dist: pandas>=2.0.0
+Requires-Dist: pytorch-lightning>=2.3.0
 Requires-Dist: scikit-misc>=0.5.0
 Requires-Dist: seaborn>=0.11.0
 Requires-Dist: torch==2.2.0

{scdataloader-1.8.1 → scdataloader-1.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scdataloader"
-version = "1.8.1"
+version = "1.9.0"
 description = "a dataloader for single cell data in lamindb"
 authors = [
     {name = "jkobject", email = "jkobject@gmail.com"}
@@ -14,7 +14,7 @@ dependencies = [
     "lamindb[bionty,ourprojects,jupyter,cellregistry,zarr]>=1.0.4,<2",
     "cellxgene-census>=0.1.0",
     "torch==2.2.0",
-    "lightning>=2.3.0",
+    "pytorch-lightning>=2.3.0",
     "anndata>=0.9.0",
     "zarr>=2.10.0",
     "matplotlib>=3.5.0",
@@ -28,6 +28,8 @@ dependencies = [
     "scikit-misc>=0.5.0",
     "palantir>=1.3.3",
     "harmonypy>=0.0.10",
+    "jupytext>=1.16.0",
+
 ]
 
 [project.optional-dependencies]

scdataloader-1.9.0/scdataloader/VERSION

@@ -0,0 +1 @@
+1.9.0

{scdataloader-1.8.1 → scdataloader-1.9.0}/scdataloader/collator.py

@@ -148,19 +148,19 @@ class Collator:
                         :, self.accepted_genes[organism_id]
                     ]
             if self.how == "most expr":
+                nnz_loc = np.where(expr > 0)[0]
                 if "knn_cells" in elem:
                     nnz_loc = np.where(expr + elem["knn_cells"].sum(0) > 0)[0]
+                    ma = self.max_len if self.max_len < len(nnz_loc) else len(nnz_loc)
+                    loc = np.argsort(expr + elem["knn_cells"].mean(0))[-(ma):][::-1]
                 else:
                     nnz_loc = np.where(expr > 0)[0]
-                ma = self.max_len if self.max_len < len(nnz_loc) else len(nnz_loc)
-                loc = np.argsort(expr)[-(ma):][::-1]
+                    ma = self.max_len if self.max_len < len(nnz_loc) else len(nnz_loc)
+                    loc = np.argsort(expr)[-(ma):][::-1]
                 # nnz_loc = [1] * 30_000
                 # loc = np.argsort(expr)[-(self.max_len) :][::-1]
             elif self.how == "random expr":
-                if "knn_cells" in elem:
-                    nnz_loc = np.where(expr + elem["knn_cells"].sum(0) > 0)[0]
-                else:
-                    nnz_loc = np.where(expr > 0)[0]
+                nnz_loc = np.where(expr > 0)[0]
                 loc = nnz_loc[
                     np.random.choice(
                         len(nnz_loc),
@@ -180,33 +180,42 @@ class Collator:
                 "some",
             ]:
                 if "knn_cells" in elem:
-                    zero_loc = np.where(expr + elem["knn_cells"].sum(0) == 0)[0]
+                    # we complete with genes expressed in the knn
+                    nnz_loc = np.where(elem["knn_cells"].sum(0) > 0)[0]
+                    ma = self.max_len if self.max_len < len(nnz_loc) else len(nnz_loc)
+                    # which is not a zero_loc in this context
+                    zero_loc = np.argsort(elem["knn_cells"].sum(0))[-(ma):][::-1]
                 else:
                     zero_loc = np.where(expr == 0)[0]
-                zero_loc = zero_loc[
-                    np.random.choice(
-                        len(zero_loc),
-                        self.add_zero_genes
-                        + (
-                            0
-                            if self.max_len < len(nnz_loc)
-                            else self.max_len - len(nnz_loc)
-                        ),
-                        replace=False,
-                    )
-                ]
+                    zero_loc = zero_loc[
+                        np.random.choice(
+                            len(zero_loc),
+                            self.add_zero_genes
+                            + (
+                                0
+                                if self.max_len < len(nnz_loc)
+                                else self.max_len - len(nnz_loc)
+                            ),
+                            replace=False,
+                        )
+                    ]
                 loc = np.concatenate((loc, zero_loc), axis=None)
-            if "knn_cells" in elem:
-                knn_cells.append(elem["knn_cells"][:, loc])
             expr = expr[loc]
-            loc = loc + self.start_idx[organism_id]
+            if "knn_cells" in elem:
+                elem["knn_cells"] = elem["knn_cells"][:, loc]
             if self.how == "some":
                 if "knn_cells" in elem:
-                    knn_cells[-1] = knn_cells[-1][self.to_subset[organism_id]]
+                    elem["knn_cells"] = elem["knn_cells"][
+                        :, self.to_subset[organism_id]
+                    ]
                 expr = expr[self.to_subset[organism_id]]
                 loc = loc[self.to_subset[organism_id]]
             exprs.append(expr)
-            gene_locs.append(loc)
+            if "knn_cells" in elem:
+                knn_cells.append(elem["knn_cells"])
+            # then we need to add the start_idx to the loc to give it the correct index
+            # according to the model
+            gene_locs.append(loc + self.start_idx[organism_id])
 
             if self.tp_name is not None:
                 tp.append(elem[self.tp_name])
@@ -243,7 +252,7 @@ class Collator:
         if len(is_meta) > 0:
             ret.update({"is_meta": Tensor(is_meta).int()})
         if len(knn_cells) > 0:
-            ret.update({"knn_cells": Tensor(knn_cells).int()})
+            ret.update({"knn_cells": Tensor(knn_cells)})
         if len(dataset) > 0:
             ret.update({"dataset": Tensor(dataset).to(long)})
         if self.downsample is not None:
@@ -251,6 +260,8 @@ class Collator:
         if self.save_output is not None:
             with open(self.save_output, "a") as f:
                 np.savetxt(f, ret["x"].numpy())
+            with open(self.save_output + "_loc", "a") as f:
+                np.savetxt(f, gene_locs)
         return ret
 
 

{scdataloader-1.8.1 → scdataloader-1.9.0}/scdataloader/config.py

@@ -118,7 +118,7 @@ MAIN_HUMAN_MOUSE_DEV_STAGE_MAP = {
     ],
     "HsapDv:0000258": [  # mature stage
         "MmusDv:0000110",  # mature stage
-        "HsapDv:0000204",
+        "HsapDv:0000204", # 
     ],
     "HsapDv:0000227": [  # late adult stage
         "MmusDv:0000091",  # 20 month-old stage

{scdataloader-1.8.1 → scdataloader-1.9.0}/scdataloader/mapped.py

@@ -428,6 +428,7 @@ class MappedCollection:
                     ],
                     dtype=int,
                 )
+                out["distances"] = distances[nn_idx]
 
         return out
 

{scdataloader-1.8.1 → scdataloader-1.9.0}/scdataloader/preprocess.py

@@ -9,7 +9,7 @@ import scanpy as sc
 from anndata import AnnData, read_h5ad
 from scipy.sparse import csr_matrix
 from upath import UPath
-
+import gc
 from scdataloader import utils as data_utils
 
 FULL_LENGTH_ASSAYS = [
@@ -18,7 +18,7 @@ FULL_LENGTH_ASSAYS = [
     "EFO:0008931",
 ]
 
-MAXFILESIZE = 10_000_000_000
+MAXFILESIZE = 5_000_000_000
 
 
 class Preprocessor:
@@ -135,6 +135,8 @@ class Preprocessor:
         self.keepdata = keepdata
 
     def __call__(self, adata, dataset_id=None) -> AnnData:
+        if self.additional_preprocess is not None:
+            adata = self.additional_preprocess(adata)
         if "organism_ontology_term_id" not in adata[0].obs.columns:
             raise ValueError(
                 "organism_ontology_term_id not found in adata.obs, you need to add an ontology term id for the organism of your anndata"
@@ -143,13 +145,11 @@ class Preprocessor:
             raise ValueError(
                 "gene names in the `var.index` field of your anndata should map to the ensembl_gene nomenclature else set `is_symbol` to True if using hugo symbols"
             )
-        if adata[0].obs.organism_ontology_term_id.iloc[0] not in self.organisms:
+        if adata.obs["organism_ontology_term_id"].iloc[0] not in self.organisms:
             raise ValueError(
                 "we cannot work with this organism",
-                adata[0].obs.organism_ontology_term_id.iloc[0],
+                adata.obs["organism_ontology_term_id"],
             )
-        if self.additional_preprocess is not None:
-            adata = self.additional_preprocess(adata)
         if adata.raw is not None and self.use_raw:
             adata.X = adata.raw.X
             del adata.raw
@@ -165,11 +165,12 @@ class Preprocessor:
                 del adata.layers
         if len(adata.varm.keys()) > 0 and not self.keepdata:
             del adata.varm
-        if len(adata.obsm.keys()) > 0 and self.do_postp and not self.keepdata:
+        if len(adata.obsm.keys()) > 0 and not self.keepdata:
             del adata.obsm
-        if len(adata.obsp.keys()) > 0 and self.do_postp and not self.keepdata:
+        if len(adata.obsp.keys()) > 0 and not self.keepdata:
             del adata.obsp
         # check that it is a count
+
         print("checking raw counts")
         if np.abs(
             adata[:50_000].X.astype(int) - adata[:50_000].X
@@ -230,23 +231,51 @@ class Preprocessor:
             )
         )
 
-        if self.is_symbol or not adata.var.index.str.contains("ENS").any():
-            if not adata.var.index.str.contains("ENS").any():
-                print("No ENS genes found, assuming gene symbols...")
-            genesdf["ensembl_gene_id"] = genesdf.index
-            var = (
-                adata.var.merge(
-                    genesdf.drop_duplicates("symbol").set_index("symbol", drop=False),
-                    left_index=True,
-                    right_index=True,
-                    how="inner",
-                )
-                .sort_values(by="ensembl_gene_id")
-                .set_index("ensembl_gene_id")
+        # Check if we have a mix of gene names and ensembl IDs
+        has_ens = adata.var.index.str.match(r"ENS.*\d{6,}$").any()
+        all_ens = adata.var.index.str.match(r"ENS.*\d{6,}$").all()
+
+        if not has_ens:
+            print("No ENS genes found, assuming gene symbols...")
+        elif not all_ens:
+            print("Mix of ENS and gene symbols found, converting all to ENS IDs...")
+
+        genesdf["ensembl_gene_id"] = genesdf.index
+
+        # For genes that are already ENS IDs, use them directly
+        ens_mask = adata.var.index.str.match(r"ENS.*\d{6,}$")
+        symbol_mask = ~ens_mask
+
+        # Handle symbol genes
+        if symbol_mask.any():
+            symbol_var = adata.var[symbol_mask].merge(
+                genesdf.drop_duplicates("symbol").set_index("symbol", drop=False),
+                left_index=True,
+                right_index=True,
+                how="inner",
+            )
+
+        # Handle ENS genes
+        if ens_mask.any():
+            ens_var = adata.var[ens_mask].merge(
+                genesdf, left_index=True, right_index=True, how="inner"
             )
-            adata = adata[:, var["symbol"]]
-            adata.var = var
-            genesdf = genesdf.set_index("ensembl_gene_id")
+
+        # Combine and sort
+        if symbol_mask.any() and ens_mask.any():
+            var = pd.concat([symbol_var, ens_var])
+        elif symbol_mask.any():
+            var = symbol_var
+        else:
+            var = ens_var
+
+        adata = adata[:, var.index]
+        var = var.sort_values(by="ensembl_gene_id").set_index("ensembl_gene_id")
+        # Update adata with combined genes
+        adata.var = var
+        genesdf = genesdf.set_index("ensembl_gene_id")
+        # Drop duplicate genes, keeping first occurrence
+        adata = adata[:, ~adata.var.index.duplicated(keep="first")]
 
         intersect_genes = set(adata.var.index).intersection(set(genesdf.index))
         print(f"Removed {len(adata.var.index) - len(intersect_genes)} genes.")
@@ -475,13 +504,17 @@ class LaminPreprocessor(Preprocessor):
                 print(file)
 
                 path = cache_path(file) if self.force_preloaded else file.cache()
-                backed = read_h5ad(path, backed="r")
-                if backed.obs.is_primary_data.sum() == 0:
-                    print(f"{file.key} only contains non primary cells.. dropping")
-                    # Save the stem_uid to a file to avoid loading it again
+                backed = file.open()
+                # backed = read_h5ad(path, backed="r")
+                if "is_primary_data" in backed.obs.columns:
+                    if backed.obs.is_primary_data.sum() == 0:
+                        print(f"{file.key} only contains non primary cells.. dropping")
+                        # Save the stem_uid to a file to avoid loading it again
                     with open("nonprimary.txt", "a") as f:
                         f.write(f"{file.stem_uid}\n")
                     continue
+                else:
+                    print("Warning: couldn't check unicity from is_primary_data column")
                 if backed.shape[1] < 1000:
                     print(
                         f"{file.key} only contains less than 1000 genes and is likely not scRNAseq... dropping"
@@ -502,16 +535,23 @@ class LaminPreprocessor(Preprocessor):
                         block_size = int(
                             (np.ceil(badata.shape[0] / 30_000) * 30_000) // num_blocks
                         )
-                        print("num blocks ", num_blocks)
+                        print(
+                            "num blocks ",
+                            num_blocks,
+                            "block size ",
+                            block_size,
+                            "total elements ",
+                            badata.shape[0],
+                        )
                         for j in range(num_blocks):
-                            if j == 0 and i == 390:
-                                continue
                             start_index = j * block_size
                             end_index = min((j + 1) * block_size, badata.shape[0])
-                            block = badata[start_index:end_index].to_memory()
+                            block = badata[start_index:end_index]
+                            block = block.to_memory()
                             print(block)
                             block = super().__call__(
-                                block, dataset_id=file.stem_uid + "_p" + str(j)
+                                block,
+                                dataset_id=file.stem_uid + "_p" + str(j),
                             )
                             myfile = ln.Artifact.from_anndata(
                                 block,
@@ -521,16 +561,19 @@ class LaminPreprocessor(Preprocessor):
                                 + " p"
                                 + str(j)
                                 + " ( revises file "
-                                + str(file.key)
+                                + str(file.stem_uid)
                                 + " )",
                                 version=version,
                             )
                             myfile.save()
+
                             if self.keep_files:
                                 files.append(myfile)
+                                del block
                             else:
                                 del myfile
                                 del block
+                            gc.collect()
 
                     else:
                         adata = super().__call__(adata, dataset_id=file.stem_uid)
@@ -543,6 +586,7 @@ class LaminPreprocessor(Preprocessor):
                         myfile.save()
                         if self.keep_files:
                             files.append(myfile)
+                            del adata
                         else:
                             del myfile
                             del adata
@@ -562,7 +606,12 @@ class LaminPreprocessor(Preprocessor):
 
                 # issues with KLlggfw6I6lvmbqiZm46
             if self.keep_files:
-                dataset = ln.Collection(files, name=name, description=description)
+                # Reconstruct collection using keys
+                dataset = ln.Collection(
+                    [ln.Artifact.filter(key=k).one() for k in files],
+                    name=name,
+                    description=description,
+                )
                 dataset.save()
                 return dataset
             else:

{scdataloader-1.8.1 → scdataloader-1.9.0}/scdataloader/utils.py

@@ -578,7 +578,6 @@ def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10
 
 
 def populate_my_ontology(
-    organisms: List[str] = ["NCBITaxon:10090", "NCBITaxon:9606"],
     sex: List[str] = ["PATO:0000384", "PATO:0000383"],
     celltypes: List[str] = [],
     ethnicities: List[str] = [],
@@ -586,7 +585,7 @@ def populate_my_ontology(
     tissues: List[str] = [],
     diseases: List[str] = [],
     dev_stages: List[str] = [],
-    organism_clade: str = "vertebrates",
+    organisms_clade: List[str] = ["vertebrates", "plants"],
 ):
     """
     creates a local version of the lamin ontologies and add the required missing values in base ontologies
@@ -622,23 +621,27 @@ def populate_my_ontology(
             ln.save(records)
         bt.CellType(name="unknown", ontology_id="unknown").save()
     # Organism
-    if organisms is not None:
-        names = (
-            bt.Organism.public(organism=organism_clade).df().index
-            if not organisms
-            else organisms
-        )
-        source = bt.PublicSource.filter(name="ensembl", organism=organism_clade).last()
-        records = [
-            organism_or_organismlist
-            if isinstance(organism_or_organismlist, bt.Organism)
-            else organism_or_organismlist[0]
-            for organism_or_organismlist in [
-                bt.Organism.from_source(ontology_id=name, source=source)
-                for name in names
+    if organisms_clade is not None:
+        records = []
+        for organism_clade in organisms_clade:
+            names = bt.Organism.public(organism=organism_clade).df().index
+            source = bt.PublicSource.filter(
+                name="ensembl", organism=organism_clade
+            ).last()
+            records += [
+                bt.Organism.from_source(name=name, source=source) for name in names
             ]
-        ]
-        ln.save(records)
+        nrecords = []
+        prevrec = set()
+        for rec in records:
+            if rec is None:
+                continue
+            if not isinstance(rec, bt.Organism):
+                rec = rec[0]
+            if rec.uid not in prevrec:
+                nrecords.append(rec)
+                prevrec.add(rec.uid)
+        ln.save(nrecords)
         bt.Organism(name="unknown", ontology_id="unknown").save()
     # Phenotype
     if sex is not None:

scdataloader-1.8.1/scdataloader/VERSION

@@ -1 +0,0 @@
-1.8.1