scdataloader 0.0.2__py3-none-any.whl → 0.0.3__py3-none-any.whl

scdataloader/utils.py

@@ -10,6 +10,7 @@ from biomart import BiomartServer
 from django.db import IntegrityError
 from scipy.sparse import csr_matrix
 from scipy.stats import median_abs_deviation
+from functools import lru_cache
 
 
 def createFoldersFor(filepath):
@@ -24,14 +25,22 @@ def createFoldersFor(filepath):
 
 
 def _fetchFromServer(ensemble_server, attributes):
+    """
+    Fetches data from the specified ensemble server.
+
+    Args:
+        ensemble_server (str): The URL of the ensemble server to fetch data from.
+        attributes (list): The list of attributes to fetch from the server.
+
+    Returns:
+        pd.DataFrame: A pandas DataFrame containing the fetched data.
+    """
     server = BiomartServer(ensemble_server)
     ensmbl = server.datasets["hsapiens_gene_ensembl"]
     print(attributes)
     res = pd.read_csv(
         io.StringIO(
-            ensmbl.search(
-                {"attributes": attributes}, header=1
-            ).content.decode()
+            ensmbl.search({"attributes": attributes}, header=1).content.decode()
         ),
         sep="\t",
     )
@@ -93,7 +102,7 @@ def getBiomartTable(
     return res
 
 
-def validate(adata, lb, organism):
+def validate(adata, organism):
     """
     validate checks if the adata object is valid for lamindb
 
@@ -116,8 +125,7 @@ def validate(adata, lb, organism):
     Returns:
         bool: True if the adata object is valid
     """
-    organism = lb.Organism.filter(ontology_id=organism).one().name
-    lb.settings.organism = organism
+    organism = bt.Organism.filter(ontology_id=organism).one().name
 
     if adata.var.index.duplicated().any():
         raise ValueError("Duplicate gene names found in adata.var.index")
@@ -136,33 +144,33 @@ def validate(adata, lb, organism):
             raise ValueError(
                 f"Column '{val}' is missing in the provided anndata object."
             )
-    bionty_source = lb.BiontySource.filter(
+    bionty_source = bt.PublicSource.filter(
         entity="DevelopmentalStage", organism=organism
     ).one()
 
-    if not lb.Ethnicity.validate(
+    if not bt.Ethnicity.validate(
         adata.obs["self_reported_ethnicity_ontology_term_id"],
         field="ontology_id",
     ).all():
         raise ValueError("Invalid ethnicity ontology term id found")
-    if not lb.Organism.validate(
+    if not bt.Organism.validate(
         adata.obs["organism_ontology_term_id"], field="ontology_id"
     ).all():
         raise ValueError("Invalid organism ontology term id found")
-    if not lb.Phenotype.validate(
+    if not bt.Phenotype.validate(
         adata.obs["sex_ontology_term_id"], field="ontology_id"
     ).all():
         raise ValueError("Invalid sex ontology term id found")
-    if not lb.Disease.validate(
+    if not bt.Disease.validate(
         adata.obs["disease_ontology_term_id"], field="ontology_id"
     ).all():
         raise ValueError("Invalid disease ontology term id found")
-    if not lb.CellType.validate(
+    if not bt.CellType.validate(
         adata.obs["cell_type_ontology_term_id"], field="ontology_id"
     ).all():
         raise ValueError("Invalid cell type ontology term id found")
     if (
-        not lb.DevelopmentalStage.filter(bionty_source=bionty_source)
+        not bt.DevelopmentalStage.filter(bionty_source=bionty_source)
         .validate(
             adata.obs["development_stage_ontology_term_id"],
             field="ontology_id",
@@ -170,16 +178,16 @@ def validate(adata, lb, organism):
         .all()
     ):
         raise ValueError("Invalid dev stage ontology term id found")
-    if not lb.Tissue.validate(
+    if not bt.Tissue.validate(
         adata.obs["tissue_ontology_term_id"], field="ontology_id"
     ).all():
         raise ValueError("Invalid tissue ontology term id found")
-    if not lb.ExperimentalFactor.validate(
+    if not bt.ExperimentalFactor.validate(
         adata.obs["assay_ontology_term_id"], field="ontology_id"
     ).all():
         raise ValueError("Invalid assay ontology term id found")
     if (
-        not lb.Gene.filter(organism=lb.settings.organism)
+        not bt.Gene.filter(organism=bt.settings.organism)
         .validate(adata.var.index, field="ensembl_gene_id")
         .all()
     ):
@@ -187,6 +195,8 @@ def validate(adata, lb, organism):
     return True
 
 
+# setting a cache of 200 elements
+# @lru_cache(maxsize=200)
 def get_all_ancestors(val, df):
     if val not in df.index:
         return set()
@@ -194,9 +204,7 @@ def get_all_ancestors(val, df):
     if parents is None or len(parents) == 0:
         return set()
     else:
-        return set.union(
-            set(parents), *[get_all_ancestors(val, df) for val in parents]
-        )
+        return set.union(set(parents), *[get_all_ancestors(val, df) for val in parents])
 
 
 def get_ancestry_mapping(all_elem, onto_df):
@@ -234,7 +242,6 @@ def get_ancestry_mapping(all_elem, onto_df):
 
 
 def load_dataset_local(
-    lb,
     remote_dataset,
     download_folder,
     name,
@@ -258,9 +265,7 @@ def load_dataset_local(
         lamindb.Dataset: The local dataset.
     """
     saved_files = []
-    default_storage = ln.Storage.filter(
-        root=ln.settings.storage.as_posix()
-    ).one()
+    default_storage = ln.Storage.filter(root=ln.settings.storage.as_posix()).one()
     files = (
         remote_dataset.artifacts.all()
         if not only
@@ -275,17 +280,15 @@ def load_dataset_local(
         if len(organism) == 0:
             print("No organism detected")
             continue
-        organism = lb.Organism.filter(ontology_id=organism[0]).one().name
-        # lb.settings.organism = organism
+        organism = bt.Organism.filter(ontology_id=organism[0]).one().name
+        # bt.settings.organism = organism
         path = file.path
         try:
             file.save()
         except IntegrityError:
             print(f"File {file.key} already exists in storage")
         # if location already has a file, don't save again
-        if use_cache and os.path.exists(
-            os.path.expanduser(download_folder + file.key)
-        ):
+        if use_cache and os.path.exists(os.path.expanduser(download_folder + file.key)):
             print(f"File {file.key} already exists in storage")
         else:
             path.download_to(download_folder + file.key)
@@ -295,13 +298,34 @@ def load_dataset_local(
         except IntegrityError:
             print(f"File {file.key} already exists in storage")
         saved_files.append(file)
-    dataset = ln.Dataset(saved_files, name=name, description=description)
+    dataset = ln.Collection(saved_files, name=name, description=description)
     dataset.save()
     return dataset
 
 
+def load_genes(organisms):
+    organismdf = []
+    if type(organisms) == str:
+        organisms = [organisms]
+    for organism in organisms:
+        genesdf = bt.Gene.filter(
+            organism_id=bt.Organism.filter(ontology_id=organism).first().id
+        ).df()
+        genesdf = genesdf[~genesdf["public_source_id"].isna()]
+        genesdf = genesdf.drop_duplicates(subset="ensembl_gene_id")
+        genesdf = genesdf.set_index("ensembl_gene_id")
+        # mitochondrial genes
+        genesdf["mt"] = genesdf.symbol.astype(str).str.startswith("MT-")
+        # ribosomal genes
+        genesdf["ribo"] = genesdf.symbol.astype(str).str.startswith(("RPS", "RPL"))
+        # hemoglobin genes.
+        genesdf["hb"] = genesdf.symbol.astype(str).str.contains(("^HB[^(P)]"))
+        genesdf["organism"] = organism
+        organismdf.append(genesdf)
+    return pd.concat(organismdf)
+
+
 def populate_my_ontology(
-    lb,
     organisms=["NCBITaxon:10090", "NCBITaxon:9606"],
     sex=["PATO:0000384", "PATO:0000383"],
     celltypes=[],
@@ -316,11 +340,11 @@ def populate_my_ontology(
 
     run this function just one for each new lamin storage
 
-    erase everything with lb.$ontology.filter().delete()
+    erase everything with bt.$ontology.filter().delete()
 
     add whatever value you need afterward like it is done here with:
 
-    `lb.$ontology(name="ddd", ontology_id="ddddd").save()`
+    `bt.$ontology(name="ddd", ontology_id="ddddd").save()`
 
     `df["assay_ontology_term_id"].unique()`
 
@@ -336,73 +360,72 @@ def populate_my_ontology(
         dev_stages (list, optional): List of developmental stages. Defaults to [].
     """
 
-    names = bt.CellType().df().index if not celltypes else celltypes
-    records = lb.CellType.from_values(names, field=lb.CellType.ontology_id)
+    names = bt.CellType.from_public().df().index if not celltypes else celltypes
+    records = bt.CellType.from_values(names, field="ontology_id")
     ln.save(records)
-    lb.CellType(name="unknown", ontology_id="unknown").save()
+    bt.CellType(name="unknown", ontology_id="unknown").save()
     # Organism
-    # names = bt.Organism().df().index if not organisms else organisms
-    # records = lb.Organism.from_values(names, field=lb.Organism.ontology_id)
-    # ln.save(records)
-    # lb.Organism(name="unknown", ontology_id="unknown").save()
+    names = bt.Organism.from_public().df().index if not organisms else organisms
+    records = [
+        i[0] if type(i) is list else i
+        for i in [bt.Organism.from_public(ontology_id=i) for i in names]
+    ]
+    ln.save(records)
+    bt.Organism(name="unknown", ontology_id="unknown").save()
     # Phenotype
-    name = bt.Phenotype().df().index if not sex else sex
-    records = lb.Phenotype.from_values(
-        name,
-        field=lb.Phenotype.ontology_id,
-        bionty_source=lb.BiontySource.filter(
-            entity="Phenotype", source="pato"
-        ).one(),
-    )
+    names = bt.Phenotype.from_public().df().index if not sex else sex
+    records = [
+        bt.Phenotype.from_public(
+            ontology_id=i,
+            public_source=bt.PublicSource.filter(
+                entity="Phenotype", source="pato"
+            ).one(),
+        )
+        for i in names
+    ]
     ln.save(records)
-    lb.Phenotype(name="unknown", ontology_id="unknown").save()
+    bt.Phenotype(name="unknown", ontology_id="unknown").save()
     # ethnicity
-    names = bt.Ethnicity().df().index if not ethnicities else ethnicities
-    records = lb.Ethnicity.from_values(names, field=lb.Ethnicity.ontology_id)
+    names = bt.Ethnicity.from_public().df().index if not ethnicities else ethnicities
+    records = bt.Ethnicity.from_values(names, field="ontology_id")
     ln.save(records)
-    lb.Ethnicity(
+    bt.Ethnicity(
         name="unknown", ontology_id="unknown"
     ).save()  # multi ethnic will have to get renamed
     # ExperimentalFactor
-    names = bt.ExperimentalFactor().df().index if not assays else assays
-    records = lb.ExperimentalFactor.from_values(
-        names, field=lb.ExperimentalFactor.ontology_id
-    )
+    names = bt.ExperimentalFactor.from_public().df().index if not assays else assays
+    records = bt.ExperimentalFactor.from_values(names, field="ontology_id")
     ln.save(records)
-    lb.ExperimentalFactor(name="unknown", ontology_id="unknown").save()
-    # lookup = lb.ExperimentalFactor.lookup()
+    bt.ExperimentalFactor(name="unknown", ontology_id="unknown").save()
+    # lookup = bt.ExperimentalFactor.lookup()
     # lookup.smart_seq_v4.parents.add(lookup.smart_like)
     # Tissue
-    names = bt.Tissue().df().index if not tissues else tissues
-    records = lb.Tissue.from_values(names, field=lb.Tissue.ontology_id)
+    names = bt.Tissue.from_public().df().index if not tissues else tissues
+    records = bt.Tissue.from_values(names, field="ontology_id")
     ln.save(records)
-    lb.Tissue(name="unknown", ontology_id="unknown").save()
+    bt.Tissue(name="unknown", ontology_id="unknown").save()
     # DevelopmentalStage
     names = (
-        bt.DevelopmentalStage().df().index if not dev_stages else dev_stages
-    )
-    records = lb.DevelopmentalStage.from_values(
-        names, field=lb.DevelopmentalStage.ontology_id
+        bt.DevelopmentalStage.from_public().df().index if not dev_stages else dev_stages
     )
+    records = bt.DevelopmentalStage.from_values(names, field="ontology_id")
     ln.save(records)
-    lb.DevelopmentalStage(name="unknown", ontology_id="unknown").save()
+    bt.DevelopmentalStage(name="unknown", ontology_id="unknown").save()
     # Disease
-    names = bt.Disease().df().index if not diseases else diseases
-    records = lb.Disease.from_values(names, field=lb.Disease.ontology_id)
+    names = bt.Disease.from_public().df().index if not diseases else diseases
+    records = bt.Disease.from_values(names, field="ontology_id")
     ln.save(records)
-    lb.Disease(name="normal", ontology_id="PATO:0000461").save()
-    lb.Disease(name="unknown", ontology_id="unknown").save()
+    bt.Disease(name="normal", ontology_id="PATO:0000461").save()
+    bt.Disease(name="unknown", ontology_id="unknown").save()
     # genes
-    for organism in organisms:
+    for organism in ["NCBITaxon:10090", "NCBITaxon:9606"]:
         # convert onto to name
-        organism = lb.Organism.filter(ontology_id=organism).one().name
-        names = bt.Gene(organism=organism).df()["ensembl_gene_id"]
-        records = lb.Gene.from_values(
+        organism = bt.Organism.filter(ontology_id=organism).one().name
+        names = bt.Gene.public(organism=organism).df()["ensembl_gene_id"]
+        records = bt.Gene.from_values(
             names,
             field="ensembl_gene_id",
-            bionty_source=lb.BiontySource.filter(
-                entity="Gene", organism=organism
-            ).first(),
+            organism=organism,
         )
         ln.save(records)
 

{scdataloader-0.0.2.dist-info → scdataloader-0.0.3.dist-info}/METADATA

@@ -1,31 +1,33 @@
 Metadata-Version: 2.1
 Name: scdataloader
-Version: 0.0.2
+Version: 0.0.3
 Summary: a dataloader for single cell data in lamindb
-Home-page: https://github.com/jkobject/scPrint
+Home-page: https://github.com/jkobject/scDataLoader
 License: GPL3
 Keywords: scRNAseq,dataloader,pytorch,lamindb,scPrint
 Author: jkobject
-Requires-Python: >=3.10,<4.0
+Requires-Python: ==3.10.*
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: anndata
 Requires-Dist: biomart
+Requires-Dist: bionty
 Requires-Dist: cellxgene-census
 Requires-Dist: decoupler
 Requires-Dist: django
 Requires-Dist: ipykernel
 Requires-Dist: lamindb
 Requires-Dist: leidenalg
+Requires-Dist: lightning
+Requires-Dist: lnschema-bionty
 Requires-Dist: matplotlib
 Requires-Dist: pandas (>=2.0.0)
+Requires-Dist: scikit-misc
 Requires-Dist: seaborn
 Requires-Dist: torch
 Requires-Dist: torchdata
-Project-URL: Repository, https://github.com/jkobject/scPrint
+Project-URL: Repository, https://github.com/jkobject/scDataLoader
 Description-Content-Type: text/markdown
 
 # scdataloader
@@ -33,7 +35,9 @@ Description-Content-Type: text/markdown
 [![codecov](https://codecov.io/gh/jkobject/scDataLoader/branch/main/graph/badge.svg?token=scDataLoader_token_here)](https://codecov.io/gh/jkobject/scDataLoader)
 [![CI](https://github.com/jkobject/scDataLoader/actions/workflows/main.yml/badge.svg)](https://github.com/jkobject/scDataLoader/actions/workflows/main.yml)
 
-Awesome single cell dataloader created by @jkobject
+Awesome single cell dataloader created by @jkobject 
+
+built on top of `lamindb` and the `.mapped()` function by Sergey: https://github.com/Koncopd 
 
 This data loader is designed to be used with:
 
@@ -51,12 +55,34 @@ It allows you to:
 3. create a more complex single cell dataset
 4. extend it to your need
 
+## About
+
+the idea is to use it to train models like scGPT / GeneFormer (and soon, scPrint ;)). It is: 
+
+1. loading from lamin 
+2. doing some dataset specific preprocessing if needed 
+3. creating a dataset object on top of .mapped() (that is needed for mapping genes, cell labels etc..)
+4. passing it to a dataloader object that can work with it correctly
+
+Currently one would have to use the preprocess function to make the dataset fit for different tools like scGPT / Geneformer. But I would want to enable it through different Collators. This is still missing and a WIP... (please do contribute!)
+
+![](docs/scdataloader.drawio.png)
+
 ## Install it from PyPI
 
 ```bash
 pip install scdataloader
 ```
 
+### Install it locally and run the notebooks:
+
+```bash
+git clone https://github.com/jkobject/scDataLoader.git
+cd scDataLoader
+poetry install
+```
+then run the notebooks with the poetry installed environment
+
 ## Usage
 
 see the notebooks in [docs](https://jkobject.github.io/scDataLoader/):

scdataloader-0.0.3.dist-info/RECORD

@@ -0,0 +1,15 @@
+scdataloader/VERSION,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+scdataloader/__init__.py,sha256=cuj9n8np6jXU05e0VzDkUQv4CYJI6StvQ0TAsURS7wg,122
+scdataloader/__main__.py,sha256=x-EDMcfJscSM5ViRZmH0ekCm7QoYRgRF7qVeNKg2Dyc,5733
+scdataloader/base.py,sha256=M1gD59OffRdLOgS1vHKygOomUoAMuzjpRtAfM3SBKF8,338
+scdataloader/collator.py,sha256=vV4kuygk_x_HthyitKvJNn1yDzcL1COMvsP8N5vaME0,9524
+scdataloader/data.py,sha256=8G-ric6pmHf1U4X_0VnTS-nKcA6ztKtrhWJwjXsmUV0,13029
+scdataloader/dataloader.py,sha256=MqASZkmu3FG0z_cIG6L_7_T1uJd5iyVtAyEgap8Fv6c,10281
+scdataloader/mapped.py,sha256=ldBgCXnbFQUlEJ7dSWFgJ0654b6e_AK41mMxAgRn1hM,12635
+scdataloader/preprocess.py,sha256=aX69Z7cDrRG0qBa1yKngMyfkLK7DvAUnUXkiKUE-bbo,22550
+scdataloader/utils.py,sha256=7RKTZIAw0fLAmG31ph0WVvzEQPyPLRUpWjjg3P53ofc,17282
+scdataloader-0.0.3.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+scdataloader-0.0.3.dist-info/METADATA,sha256=jY_1yqWY5KYiy1jiaRSvo9z5bQcmedLBaRPk5J8WHlo,38289
+scdataloader-0.0.3.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+scdataloader-0.0.3.dist-info/entry_points.txt,sha256=nLqucZaa5wiF7-1FCgMXO916WDQ9Qm0TcxQp0f1DwE4,59
+scdataloader-0.0.3.dist-info/RECORD,,

{scdataloader-0.0.2.dist-info → scdataloader-0.0.3.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.8.1
+Generator: poetry-core 1.7.0
 Root-Is-Purelib: true
 Tag: py3-none-any

scdataloader-0.0.2.dist-info/RECORD

@@ -1,12 +0,0 @@
-scdataloader/VERSION,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-scdataloader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scdataloader/base.py,sha256=M1gD59OffRdLOgS1vHKygOomUoAMuzjpRtAfM3SBKF8,338
-scdataloader/data.py,sha256=5-w4WL0Ho5RW47J37N-zdNhV4Fjs0b7lb6c6ugeTMi4,12793
-scdataloader/mapped.py,sha256=wQN2K7GnJv-UiNIlC41HItrVMW50tECAjc8mt-QV-1I,12290
-scdataloader/preprocess.py,sha256=sm5OPREZFJaGVF9VsTKGvT1jHT7sOouX_ql0mWx3_4Q,23103
-scdataloader/utils.py,sha256=Ih1LLnmRZYOpIk1IoAJKyRAT361zrgBgUhwJM04V6Pw,16115
-scdataloader-0.0.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-scdataloader-0.0.2.dist-info/METADATA,sha256=4ICXsQcdWkwrAZZVDIYG1L3d7JCpaxpr3MYlnVsD1Qw,37340
-scdataloader-0.0.2.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-scdataloader-0.0.2.dist-info/entry_points.txt,sha256=nLqucZaa5wiF7-1FCgMXO916WDQ9Qm0TcxQp0f1DwE4,59
-scdataloader-0.0.2.dist-info/RECORD,,