scdataloader 1.9.2__py3-none-any.whl → 2.0.2__py3-none-any.whl

scdataloader/utils.py

@@ -1,5 +1,7 @@
 import io
 import os
+import random
+import string
 import urllib
 from collections import Counter
 from functools import lru_cache
@@ -13,50 +15,67 @@ import torch
 from anndata import AnnData
 from biomart import BiomartServer
 from django.db import IntegrityError
+from lamindb.errors import DoesNotExist
 from scipy.sparse import csr_matrix
 from scipy.stats import median_abs_deviation
 from torch import Tensor
 
 
-def slurm_restart_count(use_mine: bool = False):
-    if use_mine:
-        return int(os.getenv("MY_SLURM_RESTART_COUNT", 0))
-    else:
-        return int(os.getenv("SLURM_RESTART_COUNT", 0))
+def fileToList(filename: str, strconv: callable = lambda x: x) -> list:
+    """
+    loads an input file with a\\n b\\n.. into a list [a,b,..]
+
+    Args:
+        input_str (str): The input string to be completed.
+
+    Returns:
+        str: The completed string with 'complete' appended.
+    """
+    with open(filename) as f:
+        return [strconv(val[:-1]) for val in f.readlines()]
 
 
-def downsample_profile(mat: Tensor, dropout: float):
+def listToFile(
+    li: List[str], filename: str, strconv: callable = lambda x: str(x)
+) -> None:
     """
-    This function downsamples the expression profile of a given single cell RNA matrix.
-
-    The noise is applied based on the renoise parameter,
-    the total counts of the matrix, and the number of genes. The function first calculates the noise
-    threshold (scaler) based on the renoise parameter. It then generates an initial matrix count by
-    applying a Poisson distribution to a random tensor scaled by the total counts and the number of genes.
-    The function then models the sampling zeros by applying a Poisson distribution to a random tensor
-    scaled by the noise threshold, the total counts, and the number of genes. The function also models
-    the technical zeros by generating a random tensor and comparing it to the noise threshold. The final
-    matrix count is calculated by subtracting the sampling zeros from the initial matrix count and
-    multiplying by the technical zeros. The function ensures that the final matrix count is not less
-    than zero by taking the maximum of the final matrix count and a tensor of zeros. The function
-    returns the final matrix count.
+    listToFile loads a list with [a,b,..] into an input file a\\n b\\n..
 
     Args:
-        mat (torch.Tensor): The input matrix.
-        dropout (float): The renoise parameter.
+        l (list): The list of elements to be written to the file.
+        filename (str): The name of the file where the list will be written.
+        strconv (callable, optional): A function to convert each element of the list to a string. Defaults to str.
 
     Returns:
-        torch.Tensor: The matrix count after applying noise.
+        None
     """
-    batch = mat.shape[0]
-    ngenes = mat.shape[1]
-    dropout = dropout * 1.1
-    # we model the sampling zeros (dropping 30% of the reads)
-    res = torch.poisson((mat * (dropout / 2))).int()
-    # we model the technical zeros (dropping 50% of the genes)
-    notdrop = (torch.rand((batch, ngenes), device=mat.device) >= (dropout / 2)).int()
-    mat = (mat - res) * notdrop
-    return torch.maximum(mat, torch.zeros((1, 1), device=mat.device, dtype=torch.int))
+    with open(filename, "w") as f:
+        for item in li:
+            f.write("%s\n" % strconv(item))
+
+
+def slurm_restart_count(use_mine: bool = False):
+    if use_mine:
+        return int(os.getenv("MY_SLURM_RESTART_COUNT", 0))
+    else:
+        return int(os.getenv("SLURM_RESTART_COUNT", 0))
+
+
+def revert_to_raw(adata, mode="logp1"):
+    res = adata.X
+    if mode == "rlogp1":
+        res = np.exp(res) - 1
+    elif mode == "logp1":
+        res = (2**res) - 1
+    elif mode == "sqrt":
+        res = (res**2) - 1
+    res = (
+        (res.T / np.array([res[i][res[i] != 0].min() for i in range(res.shape[0])]))
+        .round()
+        .T
+    )  # .sum()
+    adata.X = res
+    return adata
 
 
 def createFoldersFor(filepath: str):
@@ -71,7 +90,7 @@ def createFoldersFor(filepath: str):
 
 
 def _fetchFromServer(
-    ensemble_server: str, attributes: list, database: str = "hsapiens_gene_ensembl"
+    ensemble_server: str, attributes: List[str], database: str = "hsapiens_gene_ensembl"
 ):
     """
     Fetches data from the specified ensemble server.
@@ -248,7 +267,10 @@ def validate(adata: AnnData, organism: str, need_all=False):
     if not bt.Gene.validate(
         adata.var.index, field="ensembl_gene_id", organism=organism
     ).all():
-        raise ValueError("Invalid gene ensembl id found")
+        if not bt.Gene.validate(
+            adata.var.index, field="stable_id", organism=organism
+        ).all():
+            raise ValueError("Invalid gene ensembl id found")
     return True
 
 
@@ -274,7 +296,7 @@ def get_descendants(val, df):
     return r_onto | ontos
 
 
-def get_ancestry_mapping(all_elem: list, onto_df: pd.DataFrame):
+def get_ancestry_mapping(all_elem: List[str], onto_df: pd.DataFrame):
     """
     This function generates a mapping of all elements to their ancestors in the ontology dataframe.
 
@@ -293,7 +315,7 @@ def get_ancestry_mapping(all_elem: list, onto_df: pd.DataFrame):
     for val in ancestors.values():
         full_ancestors |= set(val)
     # removing ancestors that are not in our datasets
-    full_ancestors = full_ancestors & set(ancestors.keys())
+    # full_ancestors = full_ancestors & set(ancestors.keys())
     leafs = set(all_elem) - full_ancestors
     full_ancestors = full_ancestors - leafs
 
@@ -370,12 +392,14 @@ def load_dataset_local(
     return dataset
 
 
-def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10090",
+def load_genes(
+    organisms: Union[str, List[str]] = "NCBITaxon:9606",
+):  # "NCBITaxon:10090",
     """
     Loads genes from the given organisms.
 
     Args:
-        organisms (Union[str, list]): The organisms to load genes from.
+        organisms (Union[str, List[str]]): The organisms to load genes from.
 
     Returns:
         pd.DataFrame: The genes dataframe.
@@ -387,6 +411,9 @@ def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10
         genesdf = bt.Gene.filter(
             organism_id=bt.Organism.filter(ontology_id=organism).first().id
         ).df()
+        genesdf.loc[genesdf.ensembl_gene_id.isna(), "ensembl_gene_id"] = genesdf.loc[
+            genesdf.ensembl_gene_id.isna(), "stable_id"
+        ]
         genesdf = genesdf.drop_duplicates(subset="ensembl_gene_id")
         genesdf = genesdf.set_index("ensembl_gene_id").sort_index()
         # mitochondrial genes
@@ -408,6 +435,9 @@ def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10
         "_aux",
         "_branch_code",
         "space_id",
+        "ncbi_gene_ids",
+        "synonyms",
+        "description",
     ]:
         if col in organismdf.columns:
             organismdf.drop(columns=[col], inplace=True)
@@ -577,6 +607,54 @@ def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10
     return organismdf
 
 
+def _adding_scbasecamp_genes(
+    species=[],
+):
+    if len(species) == 0:
+        species = set(
+            bt.Organism.using("laminlabs/arc-virtual-cell-atlas").df().ontology_id
+        )
+        -set(["NCBITaxon:10090", "NCBITaxon:9606"])
+    species = list(species)
+    if len(bt.Organism.filter(ontology_id="NCBITaxon:9593")) == 0:
+        bt.Organism(
+            name="gorilla gorilla",
+            ontology_id="NCBITaxon:9593",
+            scientific_name="Gorilla gorilla gorilla",
+        ).save()
+    if len(bt.Organism.filter(ontology_id="NCBITaxon:9594")) == 0:
+        bt.Organism(
+            name="rice",
+            ontology_id="NCBITaxon:4530",
+            scientific_name="Oryza sativa (main)",
+        ).save()
+
+    for i in species:
+        print(i)
+        df = (
+            bt.Gene.using("laminlabs/arc-virtual-cell-atlas")
+            .filter(organism__ontology_id=i)
+            .all()
+            .df()
+        )
+        genes = []
+        org = bt.Organism.filter(ontology_id=i).one()
+        ido = org.id
+        for row in df.to_dict(orient="records"):
+            row["organism_id"] = ido
+            gene = bt.Gene(
+                ensembl_gene_id=row["ensembl_gene_id"],
+                stable_id=row["stable_id"],
+                description=row["description"],
+                symbol=row["symbol"],
+                biotype=row["biotype"],
+                organism=org,
+                _skip_validation=True,
+            )
+            genes.append(gene)
+        ln.save(genes, ignore_conflicts=True)
+
+
 def populate_my_ontology(
     sex: List[str] = ["PATO:0000384", "PATO:0000383"],
     celltypes: List[str] = [],
@@ -585,7 +663,8 @@ def populate_my_ontology(
     tissues: List[str] = [],
     diseases: List[str] = [],
     dev_stages: List[str] = [],
-    organisms_clade: List[str] = ["vertebrates", "plants"],
+    organisms_clade: List[str] = ["vertebrates", "plants", "metazoa"],
+    genes_from: List[str] = ["NCBITaxon:10090", "NCBITaxon:9606"],
 ):
     """
     creates a local version of the lamin ontologies and add the required missing values in base ontologies
@@ -601,8 +680,6 @@ def populate_my_ontology(
     `df["assay_ontology_term_id"].unique()`
 
     Args:
-        lb (lamindb): lamindb instance.
-        organisms (list, optional): List of organisms. Defaults to ["NCBITaxon:10090", "NCBITaxon:9606"].
         sex (list, optional): List of sexes. Defaults to ["PATO:0000384", "PATO:0000383"].
         celltypes (list, optional): List of cell types. Defaults to [].
         ethnicities (list, optional): List of ethnicities. Defaults to [].
@@ -610,6 +687,7 @@ def populate_my_ontology(
         tissues (list, optional): List of tissues. Defaults to [].
         diseases (list, optional): List of diseases. Defaults to [].
         dev_stages (list, optional): List of developmental stages. Defaults to [].
+        organisms_clade (list, optional): List of organisms clade. Defaults to ["vertebrates", "plants"].
     """
     # cell type
     if celltypes is not None:
@@ -620,17 +698,17 @@ def populate_my_ontology(
             records = bt.CellType.from_values(names, field="ontology_id")
             ln.save(records)
         bt.CellType(name="unknown", ontology_id="unknown").save()
-    # Organism
+    # OrganismClade
     if organisms_clade is not None:
         records = []
         for organism_clade in organisms_clade:
             names = bt.Organism.public(organism=organism_clade).df().index
-            source = bt.PublicSource.filter(
-                name="ensembl", organism=organism_clade
-            ).last()
-            records += [
-                bt.Organism.from_source(name=name, source=source) for name in names
-            ]
+            source = bt.Source.filter(name="ensembl", organism=organism_clade).last()
+            for name in names:
+                try:
+                    records.append(bt.Organism.from_source(name=name, source=source))
+                except DoesNotExist:
+                    print(f"Organism {name} not found in source {source}")
         nrecords = []
         prevrec = set()
         for rec in records:
@@ -646,7 +724,7 @@ def populate_my_ontology(
     # Phenotype
     if sex is not None:
         names = bt.Phenotype.public().df().index if not sex else sex
-        source = bt.PublicSource.filter(name="pato").first()
+        source = bt.Source.filter(name="pato").first()
         records = [
             bt.Phenotype.from_source(ontology_id=i, source=source) for i in names
         ]
@@ -687,7 +765,7 @@ def populate_my_ontology(
     if dev_stages is not None:
         if len(dev_stages) == 0:
             bt.DevelopmentalStage.import_source()
-            source = bt.PublicSource.filter(organism="mouse", name="mmusdv").last()
+            source = bt.Source.filter(organism="mouse", name="mmusdv").last()
             bt.DevelopmentalStage.import_source(source=source)
         else:
             names = (
@@ -710,7 +788,7 @@ def populate_my_ontology(
         bt.Disease(name="normal", ontology_id="PATO:0000461").save()
         bt.Disease(name="unknown", ontology_id="unknown").save()
     # genes
-    for organism in ["NCBITaxon:10090", "NCBITaxon:9606"]:
+    for organism in genes_from:
         # convert onto to name
         organism = bt.Organism.filter(ontology_id=organism).one().name
         names = bt.Gene.public(organism=organism).df()["ensembl_gene_id"]
@@ -727,6 +805,29 @@ def populate_my_ontology(
             ln.save(records)
 
 
+def random_str(stringLength=6, stype="all", withdigits=True):
+    """
+    Generate a random string of letters and digits
+
+    Args:
+        stringLength (int, optional): the amount of char. Defaults to 6.
+        stype (str, optional): one of lowercase, uppercase, all. Defaults to 'all'.
+        withdigits (bool, optional): digits allowed in the string? Defaults to True.
+
+        Returns:
+        str: random string
+    """
+    if stype == "lowercase":
+        lettersAndDigits = string.ascii_lowercase
+    elif stype == "uppercase":
+        lettersAndDigits = string.ascii_uppercase
+    else:
+        lettersAndDigits = string.ascii_letters
+    if withdigits:
+        lettersAndDigits += string.digits
+    return "".join(random.choice(lettersAndDigits) for i in range(stringLength))
+
+
 def is_outlier(adata: AnnData, metric: str, nmads: int):
     """
     is_outlier detects outliers in adata.obs[metric]
@@ -792,11 +893,16 @@ def translate(
         obj = bt.Disease
     elif t == "self_reported_ethnicity_ontology_term_id":
         obj = bt.Ethnicity
+    elif t == "organism_ontology_term_id":
+        obj = bt.Organism
     else:
         return None
     if type(val) is str:
         return {val: obj.filter(ontology_id=val).one().name}
-    elif type(val) is list or type(val) is set:
-        return {i: obj.filter(ontology_id=i).one().name for i in set(val)}
     elif type(val) is dict or type(val) is Counter:
         return {obj.filter(ontology_id=k).one().name: v for k, v in val.items()}
+    elif type(val) is set:
+        return {i: obj.filter(ontology_id=i).one().name for i in val}
+    else:
+        rl = {i: obj.filter(ontology_id=i).one().name for i in set(val)}
+        return [rl.get(i, None) for i in val]

{scdataloader-1.9.2.dist-info → scdataloader-2.0.2.dist-info}/METADATA

@@ -1,30 +1,29 @@
 Metadata-Version: 2.4
 Name: scdataloader
-Version: 1.9.2
+Version: 2.0.2
 Summary: a dataloader for single cell data in lamindb
 Project-URL: repository, https://github.com/jkobject/scDataLoader
 Author-email: jkobject <jkobject@gmail.com>
 License-Expression: MIT
 License-File: LICENSE
 Keywords: dataloader,lamindb,pytorch,scPRINT,scRNAseq
-Requires-Python: <3.14,>=3.10
+Requires-Python: <3.13,>=3.10
 Requires-Dist: anndata>=0.9.0
 Requires-Dist: biomart>=0.9.0
 Requires-Dist: cellxgene-census>=0.1.0
 Requires-Dist: django>=4.0.0
-Requires-Dist: harmonypy>=0.0.10
 Requires-Dist: ipykernel>=6.20.0
 Requires-Dist: jupytext>=1.16.0
-Requires-Dist: lamindb[bionty,cellregistry,jupyter,zarr]<2,>=1.0.4
+Requires-Dist: lamindb[bionty,jupyter,zarr]==1.6.2
 Requires-Dist: leidenalg>=0.8.0
+Requires-Dist: lightning>=2.3.0
 Requires-Dist: matplotlib>=3.5.0
-Requires-Dist: numpy==1.26.0
-Requires-Dist: palantir>=1.3.3
+Requires-Dist: numpy<=2.2.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pytorch-lightning>=2.3.0
 Requires-Dist: scikit-misc>=0.5.0
 Requires-Dist: seaborn>=0.11.0
-Requires-Dist: torch==2.2.0
+Requires-Dist: torch>=2.2.0
 Requires-Dist: torchdata>=0.5.0
 Requires-Dist: zarr>=2.10.0
 Provides-Extra: dev

scdataloader-2.0.2.dist-info/RECORD

@@ -0,0 +1,16 @@
+scdataloader/__init__.py,sha256=Z5HURehoWw1GrecImmTXIkv4ih8Q5RxNQWPm8zjjXOA,226
+scdataloader/__main__.py,sha256=xPOtrEpQQQZUGTnm8KTvsQcA_jR45oMG_VHqd0Ny7_M,8677
+scdataloader/base.py,sha256=M1gD59OffRdLOgS1vHKygOomUoAMuzjpRtAfM3SBKF8,338
+scdataloader/collator.py,sha256=pITHfsWUkrUW7lMfgXfs1AfekgcfW9XfGHwi9LlKwm8,13651
+scdataloader/config.py,sha256=nM8J11z2-lornryy1KxDE9675Rcxge4RGhdmpeiMhuI,7173
+scdataloader/data.json,sha256=Zb8c27yk3rwMgtAU8kkiWWAyUwYBrlCqKUyEtaAx9i8,8785
+scdataloader/data.py,sha256=aiSpw4rd5L162ox2kuD-8ujWNix5fvVlXozdlfthMNU,18176
+scdataloader/datamodule.py,sha256=pGPPuxDrWz0GPBUz_vb4FUprbuNKkjq1hjr46m-fRVU,35783
+scdataloader/mapped.py,sha256=h9YKQ8SG9tyZL8c6_Wu5Xov5ODGK6FzVuFopz58xwN4,29887
+scdataloader/preprocess.py,sha256=4iqHqeSVE-oKRvwD0KKl_QH6HQWVYSMRPo9QNSq-3Pk,39179
+scdataloader/utils.py,sha256=Z6td0cIphrYDLVrPrV8q4jUC_HtwGQmi-NcbpdbWrns,31034
+scdataloader-2.0.2.dist-info/METADATA,sha256=QsyNBOyn_U9_TjFVbN-5WIGIkmKWf2sSksZUzKcNlqE,10314
+scdataloader-2.0.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+scdataloader-2.0.2.dist-info/entry_points.txt,sha256=VXAN1m_CjbdLJ6SKYR0sBLGDV4wvv31ri7fWWuwbpno,60
+scdataloader-2.0.2.dist-info/licenses/LICENSE,sha256=rGy_eYmnxtbOvKs7qt5V0czSWxJwgX_MlgMyTZwDHbc,1073
+scdataloader-2.0.2.dist-info/RECORD,,

{scdataloader-1.9.2.dist-info → scdataloader-2.0.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

scdataloader-2.0.2.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Jérémie Kalfon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scdataloader/VERSION

@@ -1 +0,0 @@
-1.9.2

scdataloader-1.9.2.dist-info/RECORD

@@ -1,16 +0,0 @@
-scdataloader/VERSION,sha256=u1mRkBjudkgsI_HORCGc5MnGOTxl7w3a5A0y151BO7U,6
-scdataloader/__init__.py,sha256=Z5HURehoWw1GrecImmTXIkv4ih8Q5RxNQWPm8zjjXOA,226
-scdataloader/__main__.py,sha256=3aZnqYrH8XDT9nW9Dbb3o9kr-sx1STmXDQHxBo_h_q0,8719
-scdataloader/base.py,sha256=M1gD59OffRdLOgS1vHKygOomUoAMuzjpRtAfM3SBKF8,338
-scdataloader/collator.py,sha256=qb1SDQ358R4w56cxOXvVLpodlZpGfVDCXocFIhqpJ0I,12867
-scdataloader/config.py,sha256=YQUKCyYTg4wTseBWumPDHKtmqI7DMR-zu5FPJUWkG-c,6549
-scdataloader/data.py,sha256=xWlNU6cJmrzP4BFMsJDIksLaxe1pUfgDBlQ_IeLIXj0,15578
-scdataloader/datamodule.py,sha256=MaBSH0MqZdDbwiGnCM4xjz0KF05WT00sycJStS7GL5w,19786
-scdataloader/mapped.py,sha256=qzhGYQ2S3IDXnnO1EM1wO_png5lDiOtfuDeYc1pQaXg,27303
-scdataloader/preprocess.py,sha256=LNXlP80rj8Ze2ElyIgLuF9x_lNA78jAI-seGMOyMKGs,37496
-scdataloader/utils.py,sha256=7ycZoV01Gn3WDHOTmXqxMXlzBPSfYtjc9NbGI7gjdwI,28445
-scdataloader-1.9.2.dist-info/METADATA,sha256=jLx2v-SJTAv-jsUjmgLNIrTenwtnfUHTnKYuy9hI3lg,10363
-scdataloader-1.9.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-scdataloader-1.9.2.dist-info/entry_points.txt,sha256=VXAN1m_CjbdLJ6SKYR0sBLGDV4wvv31ri7fWWuwbpno,60
-scdataloader-1.9.2.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-scdataloader-1.9.2.dist-info/RECORD,,