scdataloader 1.9.2__py3-none-any.whl → 2.0.0__py3-none-any.whl

scdataloader/utils.py

@@ -18,45 +18,42 @@ from scipy.stats import median_abs_deviation
 from torch import Tensor
 
 
-def slurm_restart_count(use_mine: bool = False):
-    if use_mine:
-        return int(os.getenv("MY_SLURM_RESTART_COUNT", 0))
-    else:
-        return int(os.getenv("SLURM_RESTART_COUNT", 0))
+def fileToList(filename: str, strconv: callable = lambda x: x) -> list:
+    """
+    loads an input file with a\\n b\\n.. into a list [a,b,..]
 
+    Args:
+        input_str (str): The input string to be completed.
 
-def downsample_profile(mat: Tensor, dropout: float):
+    Returns:
+        str: The completed string with 'complete' appended.
     """
-    This function downsamples the expression profile of a given single cell RNA matrix.
-
-    The noise is applied based on the renoise parameter,
-    the total counts of the matrix, and the number of genes. The function first calculates the noise
-    threshold (scaler) based on the renoise parameter. It then generates an initial matrix count by
-    applying a Poisson distribution to a random tensor scaled by the total counts and the number of genes.
-    The function then models the sampling zeros by applying a Poisson distribution to a random tensor
-    scaled by the noise threshold, the total counts, and the number of genes. The function also models
-    the technical zeros by generating a random tensor and comparing it to the noise threshold. The final
-    matrix count is calculated by subtracting the sampling zeros from the initial matrix count and
-    multiplying by the technical zeros. The function ensures that the final matrix count is not less
-    than zero by taking the maximum of the final matrix count and a tensor of zeros. The function
-    returns the final matrix count.
+    with open(filename) as f:
+        return [strconv(val[:-1]) for val in f.readlines()]
+
+
+def listToFile(li: list, filename: str, strconv: callable = lambda x: str(x)) -> None:
+    """
+    listToFile loads a list with [a,b,..] into an input file a\\n b\\n..
 
     Args:
-        mat (torch.Tensor): The input matrix.
-        dropout (float): The renoise parameter.
+        l (list): The list of elements to be written to the file.
+        filename (str): The name of the file where the list will be written.
+        strconv (callable, optional): A function to convert each element of the list to a string. Defaults to str.
 
     Returns:
-        torch.Tensor: The matrix count after applying noise.
+        None
     """
-    batch = mat.shape[0]
-    ngenes = mat.shape[1]
-    dropout = dropout * 1.1
-    # we model the sampling zeros (dropping 30% of the reads)
-    res = torch.poisson((mat * (dropout / 2))).int()
-    # we model the technical zeros (dropping 50% of the genes)
-    notdrop = (torch.rand((batch, ngenes), device=mat.device) >= (dropout / 2)).int()
-    mat = (mat - res) * notdrop
-    return torch.maximum(mat, torch.zeros((1, 1), device=mat.device, dtype=torch.int))
+    with open(filename, "w") as f:
+        for item in li:
+            f.write("%s\n" % strconv(item))
+
+
+def slurm_restart_count(use_mine: bool = False):
+    if use_mine:
+        return int(os.getenv("MY_SLURM_RESTART_COUNT", 0))
+    else:
+        return int(os.getenv("SLURM_RESTART_COUNT", 0))
 
 
 def createFoldersFor(filepath: str):
@@ -248,7 +245,10 @@ def validate(adata: AnnData, organism: str, need_all=False):
     if not bt.Gene.validate(
         adata.var.index, field="ensembl_gene_id", organism=organism
     ).all():
-        raise ValueError("Invalid gene ensembl id found")
+        if not bt.Gene.validate(
+            adata.var.index, field="stable_id", organism=organism
+        ).all():
+            raise ValueError("Invalid gene ensembl id found")
     return True
 
 
@@ -293,7 +293,7 @@ def get_ancestry_mapping(all_elem: list, onto_df: pd.DataFrame):
     for val in ancestors.values():
         full_ancestors |= set(val)
     # removing ancestors that are not in our datasets
-    full_ancestors = full_ancestors & set(ancestors.keys())
+    # full_ancestors = full_ancestors & set(ancestors.keys())
     leafs = set(all_elem) - full_ancestors
     full_ancestors = full_ancestors - leafs
 
@@ -387,6 +387,9 @@ def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10
         genesdf = bt.Gene.filter(
             organism_id=bt.Organism.filter(ontology_id=organism).first().id
         ).df()
+        genesdf.loc[genesdf.ensembl_gene_id.isna(), "ensembl_gene_id"] = genesdf.loc[
+            genesdf.ensembl_gene_id.isna(), "stable_id"
+        ]
         genesdf = genesdf.drop_duplicates(subset="ensembl_gene_id")
         genesdf = genesdf.set_index("ensembl_gene_id").sort_index()
         # mitochondrial genes
@@ -408,6 +411,9 @@ def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10
         "_aux",
         "_branch_code",
         "space_id",
+        "ncbi_gene_ids",
+        "synonyms",
+        "description",
     ]:
         if col in organismdf.columns:
             organismdf.drop(columns=[col], inplace=True)

{scdataloader-1.9.2.dist-info → scdataloader-2.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdataloader
-Version: 1.9.2
+Version: 2.0.0
 Summary: a dataloader for single cell data in lamindb
 Project-URL: repository, https://github.com/jkobject/scDataLoader
 Author-email: jkobject <jkobject@gmail.com>
@@ -12,14 +12,13 @@ Requires-Dist: anndata>=0.9.0
 Requires-Dist: biomart>=0.9.0
 Requires-Dist: cellxgene-census>=0.1.0
 Requires-Dist: django>=4.0.0
-Requires-Dist: harmonypy>=0.0.10
 Requires-Dist: ipykernel>=6.20.0
 Requires-Dist: jupytext>=1.16.0
-Requires-Dist: lamindb[bionty,cellregistry,jupyter,zarr]<2,>=1.0.4
+Requires-Dist: lamindb[bionty,cellregistry,jupyter,zarr]==1.0.4
 Requires-Dist: leidenalg>=0.8.0
+Requires-Dist: lightning>=2.3.0
 Requires-Dist: matplotlib>=3.5.0
 Requires-Dist: numpy==1.26.0
-Requires-Dist: palantir>=1.3.3
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pytorch-lightning>=2.3.0
 Requires-Dist: scikit-misc>=0.5.0

scdataloader-2.0.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+scdataloader/__init__.py,sha256=Z5HURehoWw1GrecImmTXIkv4ih8Q5RxNQWPm8zjjXOA,226
+scdataloader/__main__.py,sha256=3aZnqYrH8XDT9nW9Dbb3o9kr-sx1STmXDQHxBo_h_q0,8719
+scdataloader/base.py,sha256=M1gD59OffRdLOgS1vHKygOomUoAMuzjpRtAfM3SBKF8,338
+scdataloader/collator.py,sha256=pC5PVvxwyE7e84cdr5YC4ae85NbJubk6bfldIfOLFNE,12039
+scdataloader/config.py,sha256=nM8J11z2-lornryy1KxDE9675Rcxge4RGhdmpeiMhuI,7173
+scdataloader/data.json,sha256=Zb8c27yk3rwMgtAU8kkiWWAyUwYBrlCqKUyEtaAx9i8,8785
+scdataloader/data.py,sha256=skUBLX07cxC9gl4X0zDQ3rUpm0EJoUjngvdhL4lo5IA,18633
+scdataloader/datamodule.py,sha256=fKYjQlli9wm3uCJbD6FedooNSorjl5vfg5LuwDCQkQU,34388
+scdataloader/mapped.py,sha256=0erd1vCfCnUkdMuFO4Md0d78gNeNaHSWwRVXN4zhtQQ,29802
+scdataloader/preprocess.py,sha256=CA0pvfcfZ_nMhso7vl9MfKhKfaJbYwANqHfsa5Vyco4,38425
+scdataloader/utils.py,sha256=YzTCV1IkfXIaQmtdTXJvo_Vj1l_Dhau7UUM_BBccvL0,27939
+scdataloader-2.0.0.dist-info/METADATA,sha256=rNBXB06aOnCNTu8LjijBTReTototB2tRnGmnHaFtyVE,10328
+scdataloader-2.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+scdataloader-2.0.0.dist-info/entry_points.txt,sha256=VXAN1m_CjbdLJ6SKYR0sBLGDV4wvv31ri7fWWuwbpno,60
+scdataloader-2.0.0.dist-info/licenses/LICENSE,sha256=rGy_eYmnxtbOvKs7qt5V0czSWxJwgX_MlgMyTZwDHbc,1073
+scdataloader-2.0.0.dist-info/RECORD,,

scdataloader-2.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Jérémie Kalfon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scdataloader/VERSION

@@ -1 +0,0 @@
-1.9.2

scdataloader-1.9.2.dist-info/RECORD

@@ -1,16 +0,0 @@
-scdataloader/VERSION,sha256=u1mRkBjudkgsI_HORCGc5MnGOTxl7w3a5A0y151BO7U,6
-scdataloader/__init__.py,sha256=Z5HURehoWw1GrecImmTXIkv4ih8Q5RxNQWPm8zjjXOA,226
-scdataloader/__main__.py,sha256=3aZnqYrH8XDT9nW9Dbb3o9kr-sx1STmXDQHxBo_h_q0,8719
-scdataloader/base.py,sha256=M1gD59OffRdLOgS1vHKygOomUoAMuzjpRtAfM3SBKF8,338
-scdataloader/collator.py,sha256=qb1SDQ358R4w56cxOXvVLpodlZpGfVDCXocFIhqpJ0I,12867
-scdataloader/config.py,sha256=YQUKCyYTg4wTseBWumPDHKtmqI7DMR-zu5FPJUWkG-c,6549
-scdataloader/data.py,sha256=xWlNU6cJmrzP4BFMsJDIksLaxe1pUfgDBlQ_IeLIXj0,15578
-scdataloader/datamodule.py,sha256=MaBSH0MqZdDbwiGnCM4xjz0KF05WT00sycJStS7GL5w,19786
-scdataloader/mapped.py,sha256=qzhGYQ2S3IDXnnO1EM1wO_png5lDiOtfuDeYc1pQaXg,27303
-scdataloader/preprocess.py,sha256=LNXlP80rj8Ze2ElyIgLuF9x_lNA78jAI-seGMOyMKGs,37496
-scdataloader/utils.py,sha256=7ycZoV01Gn3WDHOTmXqxMXlzBPSfYtjc9NbGI7gjdwI,28445
-scdataloader-1.9.2.dist-info/METADATA,sha256=jLx2v-SJTAv-jsUjmgLNIrTenwtnfUHTnKYuy9hI3lg,10363
-scdataloader-1.9.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-scdataloader-1.9.2.dist-info/entry_points.txt,sha256=VXAN1m_CjbdLJ6SKYR0sBLGDV4wvv31ri7fWWuwbpno,60
-scdataloader-1.9.2.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-scdataloader-1.9.2.dist-info/RECORD,,