scdataloader 1.6.3__py3-none-any.whl → 1.7.0__py3-none-any.whl

scdataloader/utils.py

@@ -147,7 +147,7 @@ def getBiomartTable(
     return res
 
 
-def validate(adata: AnnData, organism: str):
+def validate(adata: AnnData, organism: str, need_all=True):
     """
     validate checks if the adata object is valid for lamindb
 
@@ -185,7 +185,7 @@ def validate(adata: AnnData, organism: str):
         "tissue_ontology_term_id",
         "assay_ontology_term_id",
     ]:
-        if val not in adata.obs.columns:
+        if val not in adata.obs.columns and need_all:
             raise ValueError(
                 f"Column '{val}' is missing in the provided anndata object."
             )
@@ -193,7 +193,9 @@ def validate(adata: AnnData, organism: str):
     if not bt.Ethnicity.validate(
         adata.obs["self_reported_ethnicity_ontology_term_id"],
         field="ontology_id",
-    ).all():
+    ).all() and not set(adata.obs["self_reported_ethnicity_ontology_term_id"]) == set(
+        ["unknown"]
+    ):
         raise ValueError("Invalid ethnicity ontology term id found")
     if not bt.Organism.validate(
         adata.obs["organism_ontology_term_id"], field="ontology_id"
@@ -201,28 +203,40 @@ def validate(adata: AnnData, organism: str):
         raise ValueError("Invalid organism ontology term id found")
     if not bt.Phenotype.validate(
         adata.obs["sex_ontology_term_id"], field="ontology_id"
-    ).all():
+    ).all() and not set(adata.obs["self_reported_ethnicity_ontology_term_id"]) == set(
+        ["unknown"]
+    ):
         raise ValueError("Invalid sex ontology term id found")
     if not bt.Disease.validate(
         adata.obs["disease_ontology_term_id"], field="ontology_id"
-    ).all():
+    ).all() and not set(adata.obs["self_reported_ethnicity_ontology_term_id"]) == set(
+        ["unknown"]
+    ):
         raise ValueError("Invalid disease ontology term id found")
     if not bt.CellType.validate(
         adata.obs["cell_type_ontology_term_id"], field="ontology_id"
-    ).all():
+    ).all() and not set(adata.obs["self_reported_ethnicity_ontology_term_id"]) == set(
+        ["unknown"]
+    ):
         raise ValueError("Invalid cell type ontology term id found")
     if not bt.DevelopmentalStage.validate(
         adata.obs["development_stage_ontology_term_id"],
         field="ontology_id",
-    ).all():
+    ).all() and not set(adata.obs["self_reported_ethnicity_ontology_term_id"]) == set(
+        ["unknown"]
+    ):
         raise ValueError("Invalid dev stage ontology term id found")
     if not bt.Tissue.validate(
         adata.obs["tissue_ontology_term_id"], field="ontology_id"
-    ).all():
+    ).all() and not set(adata.obs["self_reported_ethnicity_ontology_term_id"]) == set(
+        ["unknown"]
+    ):
         raise ValueError("Invalid tissue ontology term id found")
     if not bt.ExperimentalFactor.validate(
         adata.obs["assay_ontology_term_id"], field="ontology_id"
-    ).all():
+    ).all() and not set(adata.obs["self_reported_ethnicity_ontology_term_id"]) == set(
+        ["unknown"]
+    ):
         raise ValueError("Invalid assay ontology term id found")
     if not bt.Gene.validate(
         adata.var.index, field="ensembl_gene_id", organism=organism
@@ -378,6 +392,169 @@ def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10
     ]:
         if col in organismdf.columns:
             organismdf.drop(columns=[col], inplace=True)
+    # temp fix
+    drop = {
+        "ENSG00000112096",
+        "ENSG00000137808",
+        "ENSG00000161149",
+        "ENSG00000182230",
+        "ENSG00000203812",
+        "ENSG00000204092",
+        "ENSG00000205485",
+        "ENSG00000212951",
+        "ENSG00000215271",
+        "ENSG00000221995",
+        "ENSG00000224739",
+        "ENSG00000224745",
+        "ENSG00000225178",
+        "ENSG00000225932",
+        "ENSG00000226377",
+        "ENSG00000226380",
+        "ENSG00000226403",
+        "ENSG00000227021",
+        "ENSG00000227220",
+        "ENSG00000227902",
+        "ENSG00000228139",
+        "ENSG00000228206",
+        "ENSG00000228906",
+        "ENSG00000229352",
+        "ENSG00000231575",
+        "ENSG00000232196",
+        "ENSG00000232295",
+        "ENSG00000233776",
+        "ENSG00000236166",
+        "ENSG00000236673",
+        "ENSG00000236740",
+        "ENSG00000236886",
+        "ENSG00000236996",
+        "ENSG00000237133",
+        "ENSG00000237513",
+        "ENSG00000237548",
+        "ENSG00000237838",
+        "ENSG00000239446",
+        "ENSG00000239467",
+        "ENSG00000239665",
+        "ENSG00000244693",
+        "ENSG00000244952",
+        "ENSG00000249860",
+        "ENSG00000251044",
+        "ENSG00000253878",
+        "ENSG00000254561",
+        "ENSG00000254740",
+        "ENSG00000255633",
+        "ENSG00000255823",
+        "ENSG00000256045",
+        "ENSG00000256222",
+        "ENSG00000256374",
+        "ENSG00000256427",
+        "ENSG00000256618",
+        "ENSG00000256863",
+        "ENSG00000256892",
+        "ENSG00000258414",
+        "ENSG00000258808",
+        "ENSG00000258861",
+        "ENSG00000259444",
+        "ENSG00000259820",
+        "ENSG00000259834",
+        "ENSG00000259855",
+        "ENSG00000260461",
+        "ENSG00000261068",
+        "ENSG00000261438",
+        "ENSG00000261490",
+        "ENSG00000261534",
+        "ENSG00000261737",
+        "ENSG00000261773",
+        "ENSG00000261963",
+        "ENSG00000262668",
+        "ENSG00000263464",
+        "ENSG00000267637",
+        "ENSG00000268955",
+        "ENSG00000269028",
+        "ENSG00000269900",
+        "ENSG00000269933",
+        "ENSG00000269966",
+        "ENSG00000270188",
+        "ENSG00000270394",
+        "ENSG00000270672",
+        "ENSG00000271043",
+        "ENSG00000271409",
+        "ENSG00000271734",
+        "ENSG00000271870",
+        "ENSG00000272040",
+        "ENSG00000272196",
+        "ENSG00000272267",
+        "ENSG00000272354",
+        "ENSG00000272370",
+        "ENSG00000272551",
+        "ENSG00000272567",
+        "ENSG00000272880",
+        "ENSG00000272904",
+        "ENSG00000272934",
+        "ENSG00000273301",
+        "ENSG00000273370",
+        "ENSG00000273496",
+        "ENSG00000273576",
+        "ENSG00000273614",
+        "ENSG00000273837",
+        "ENSG00000273888",
+        "ENSG00000273923",
+        "ENSG00000276612",
+        "ENSG00000276814",
+        "ENSG00000277050",
+        "ENSG00000277077",
+        "ENSG00000277352",
+        "ENSG00000277666",
+        "ENSG00000277761",
+        "ENSG00000278198",
+        "ENSG00000278782",
+        "ENSG00000278927",
+        "ENSG00000278955",
+        "ENSG00000279226",
+        "ENSG00000279765",
+        "ENSG00000279769",
+        "ENSG00000279948",
+        "ENSG00000280058",
+        "ENSG00000280095",
+        "ENSG00000280250",
+        "ENSG00000280346",
+        "ENSG00000280374",
+        "ENSG00000280710",
+        "ENSG00000282080",
+        "ENSG00000282246",
+        "ENSG00000282965",
+        "ENSG00000283486",
+        "ENSG00000284299",
+        "ENSG00000284741",
+        "ENSG00000285106",
+        "ENSG00000285162",
+        "ENSG00000285476",
+        "ENSG00000285762",
+        "ENSG00000286065",
+        "ENSG00000286228",
+        "ENSG00000286601",
+        "ENSG00000286699",
+        "ENSG00000286949",
+        "ENSG00000286996",
+        "ENSG00000287116",
+        "ENSG00000287388",
+        "ENSG00000288541",
+        "ENSG00000288546",
+        "ENSG00000288630",
+        "ENSG00000288639",
+        "ENSMUSG00000069518",
+        "ENSMUSG00000073682",
+        "ENSMUSG00000075014",
+        "ENSMUSG00000075015",
+        "ENSMUSG00000078091",
+        "ENSMUSG00000094958",
+        "ENSMUSG00000095547",
+        "ENSMUSG00000095891",
+        "ENSMUSG00000096385",
+        "ENSMUSG00000096519",
+        "ENSMUSG00000096923",
+        "ENSMUSG00000097078",
+    }
+    organismdf = organismdf[~organismdf.index.isin(drop)]
     return organismdf
 
 
@@ -419,7 +596,7 @@ def populate_my_ontology(
     # cell type
     if celltypes is not None:
         if len(celltypes) == 0:
-            bt.CellType.import_from_source(update=True)
+            bt.CellType.import_source()
         else:
             names = bt.CellType.public().df().index if not celltypes else celltypes
             records = bt.CellType.from_values(names, field="ontology_id")
@@ -434,9 +611,9 @@ def populate_my_ontology(
         )
         source = bt.PublicSource.filter(name="ensembl", organism=organism_clade).last()
         records = [
-            i[0] if type(i) is list else i
-            for i in [
-                bt.Organism.from_source(ontology_id=i, source=source) for i in names
+            organism_or_organismlist if isinstance(organism_or_organismlist, bt.Organism) else organism_or_organismlist[0]
+            for organism_or_organismlist in [
+                bt.Organism.from_source(ontology_id=name, source=source) for name in names
             ]
         ]
         ln.save(records)
@@ -453,7 +630,7 @@ def populate_my_ontology(
     # ethnicity
     if ethnicities is not None:
         if len(ethnicities) == 0:
-            bt.Ethnicity.import_from_source(update=True)
+            bt.Ethnicity.import_source()
         else:
             names = bt.Ethnicity.public().df().index if not ethnicities else ethnicities
             records = bt.Ethnicity.from_values(names, field="ontology_id")
@@ -464,7 +641,7 @@ def populate_my_ontology(
     # ExperimentalFactor
     if assays is not None:
         if len(assays) == 0:
-            bt.ExperimentalFactor.import_from_source(update=True)
+            bt.ExperimentalFactor.import_source()
         else:
             names = bt.ExperimentalFactor.public().df().index if not assays else assays
             records = bt.ExperimentalFactor.from_values(names, field="ontology_id")
@@ -475,7 +652,7 @@ def populate_my_ontology(
     # Tissue
     if tissues is not None:
         if len(tissues) == 0:
-            bt.Tissue.import_from_source(update=True)
+            bt.Tissue.import_source()
         else:
             names = bt.Tissue.public().df().index if not tissues else tissues
             records = bt.Tissue.from_values(names, field="ontology_id")
@@ -484,9 +661,9 @@ def populate_my_ontology(
     # DevelopmentalStage
     if dev_stages is not None:
         if len(dev_stages) == 0:
-            bt.DevelopmentalStage.import_from_source(update=True)
+            bt.DevelopmentalStage.import_source()
             source = bt.PublicSource.filter(organism="mouse", name="mmusdv").last()
-            bt.DevelopmentalStage.import_from_source(source=source)
+            bt.DevelopmentalStage.import_source(source=source)
         else:
             names = (
                 bt.DevelopmentalStage.public().df().index
@@ -500,7 +677,7 @@ def populate_my_ontology(
     # Disease
     if diseases is not None:
         if len(diseases) == 0:
-            bt.Disease.import_from_source(update=True)
+            bt.Disease.import_source()
         else:
             names = bt.Disease.public().df().index if not diseases else diseases
             records = bt.Disease.from_values(names, field="ontology_id")
@@ -575,18 +752,26 @@ def translate(
         dict: the mapping for the translation
     """
     if t == "cell_type_ontology_term_id":
-        obj = bt.CellType.public(organism="all")
+        obj = bt.CellType
     elif t == "assay_ontology_term_id":
-        obj = bt.ExperimentalFactor.public()
+        obj = bt.ExperimentalFactor
     elif t == "tissue_ontology_term_id":
-        obj = bt.Tissue.public()
+        obj = bt.Tissue
+    elif t in [
+        "development_stage_ontology_term_id",
+        "simplified_dev_stage",
+        "age_group",
+    ]:
+        obj = bt.DevelopmentalStage
+    elif t == "disease_ontology_term_id":
+        obj = bt.Disease
+    elif t == "self_reported_ethnicity_ontology_term_id":
+        obj = bt.Ethnicity
     else:
         return None
     if type(val) is str:
-        return {val: obj.search(val, field=obj.ontology_id).name.iloc[0]}
+        return {val: obj.filter(ontology_id=val).one().name}
     elif type(val) is list or type(val) is set:
-        return {i: obj.search(i, field=obj.ontology_id).name.iloc[0] for i in set(val)}
+        return {i: obj.filter(ontology_id=i).one().name for i in set(val)}
     elif type(val) is dict or type(val) is Counter:
-        return {
-            obj.search(k, field=obj.ontology_id).name.iloc[0]: v for k, v in val.items()
-        }
+        return {obj.filter(ontology_id=k).one().name: v for k, v in val.items()}

{scdataloader-1.6.3.dist-info → scdataloader-1.7.0.dist-info}/METADATA

@@ -1,23 +1,24 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: scdataloader
-Version: 1.6.3
+Version: 1.7.0
 Summary: a dataloader for single cell data in lamindb
 Project-URL: repository, https://github.com/jkobject/scDataLoader
 Author-email: jkobject <jkobject@gmail.com>
-License: MIT
+License-Expression: MIT
+License-File: LICENSE
 Keywords: dataloader,lamindb,pytorch,scPRINT,scRNAseq
-Requires-Python: <3.11,>=3.10
+Requires-Python: <3.14,>=3.10
 Requires-Dist: anndata>=0.9.0
 Requires-Dist: biomart>=0.9.0
 Requires-Dist: cellxgene-census>=0.1.0
 Requires-Dist: django>=4.0.0
 Requires-Dist: harmonypy>=0.0.10
 Requires-Dist: ipykernel>=6.20.0
-Requires-Dist: lamindb[bionty]==0.76.12
+Requires-Dist: lamindb[bionty]==0.77.2
 Requires-Dist: leidenalg>=0.8.0
 Requires-Dist: lightning>=2.0.0
 Requires-Dist: matplotlib>=3.5.0
-Requires-Dist: numpy>=1.26.0
+Requires-Dist: numpy==1.26.0
 Requires-Dist: palantir>=1.3.3
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: scikit-misc>=0.5.0
@@ -50,6 +51,8 @@ Description-Content-Type: text/markdown
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![DOI](https://img.shields.io/badge/DOI-10.1101%2F2024.07.29.605556-blue)](https://doi.org/10.1101/2024.07.29.605556)
 
+<img src="scdataloader.png" width="600">
+
 This single cell pytorch dataloader / lighting datamodule is designed to be used with:
 
 - [lamindb](https://lamin.ai/)

scdataloader-1.7.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+scdataloader/VERSION,sha256=u3Mg2DHnoVGqkBw15zJsdS-i71Ak8wdoxCMZuL7Rce0,6
+scdataloader/__init__.py,sha256=4sSZSnNM-gtyiB28M_FM3o8lNabmsofct9SWWry1_zA,170
+scdataloader/__main__.py,sha256=CcvUnvgnF2d1QQHjkIhhzeK9vgplbhdHiGMawmxhq6g,7454
+scdataloader/base.py,sha256=M1gD59OffRdLOgS1vHKygOomUoAMuzjpRtAfM3SBKF8,338
+scdataloader/collator.py,sha256=NmbMAxkFZLufWpn0yBY6d1me2nUKdV0VG11Js8rgghU,11560
+scdataloader/config.py,sha256=tu9hkUiU2HfaIiVzdmrjbzt73yV4zP-t8lDuJqyGcDA,6546
+scdataloader/data.py,sha256=K0r_RlLBza3WsWQVzybZjskKDfwFe8qMqLcJwdZ1yuw,15172
+scdataloader/datamodule.py,sha256=-GumOkOXDn7DJnqo2yhmPpEcIZUtw0LulFOnl3nkouw,20193
+scdataloader/mapped.py,sha256=u3vo7vcE4Q72qY0j7uHpZvlTTYr4yc3RaRrwE7AAhaE,27122
+scdataloader/preprocess.py,sha256=feaXGQYNfChbISZCWCnIZL1qwmzfwmNygbL-xVTwC0o,34595
+scdataloader/utils.py,sha256=MRuqbRcCkb45k_G4QCwog0C6-Az4ZcklVPn47aZJLGs,27870
+scdataloader-1.7.0.dist-info/METADATA,sha256=r0oXvOe1kqoRlbYJim4MTqgRADjP3t_xDaxXGrGomkM,9907
+scdataloader-1.7.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+scdataloader-1.7.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+scdataloader-1.7.0.dist-info/RECORD,,

{scdataloader-1.6.3.dist-info → scdataloader-1.7.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.26.3
+Generator: hatchling 1.27.0
 Root-Is-Purelib: true
 Tag: py3-none-any

scdataloader-1.6.3.dist-info/RECORD

@@ -1,14 +0,0 @@
-scdataloader/VERSION,sha256=hvj6gyp2NYIB1uL88LtHcn7-LbI69zDbZM6tZSd3a-o,6
-scdataloader/__init__.py,sha256=5y9VzRhOAUWeYMn2MrRRRlzgdiMjRFytr7gcn-I6IkE,147
-scdataloader/__main__.py,sha256=VXrt2IykBypnIXWydwA7NfF7LtRGc-0Khjtm5OIBNpI,6527
-scdataloader/base.py,sha256=M1gD59OffRdLOgS1vHKygOomUoAMuzjpRtAfM3SBKF8,338
-scdataloader/collator.py,sha256=gzHiuixUwK8JClhAbG12kgWMU_VTKkowibA-tDFpbwo,11341
-scdataloader/config.py,sha256=rrW2DZxG4J2_pmpDbXXsaKJkpNC57w5dIlItiFbANYw,2905
-scdataloader/data.py,sha256=3dCp-lIAfOkCi76SH5W3iSqFmAWZslwARkN9v5mylz8,14907
-scdataloader/datamodule.py,sha256=B-udBevPSPF__hfy0pOz1dGovgE95K2pxPupjB7RblI,16936
-scdataloader/preprocess.py,sha256=pH4EPrcRqH34o3t5X3A4kETiYdCZngih5SdP_PPfgOo,29178
-scdataloader/utils.py,sha256=7tgt3sPj_XTKb-UlJDAZWvQr0_DG9VTC6ioiLdBWFFE,22498
-scdataloader-1.6.3.dist-info/METADATA,sha256=iBh6pruWqZArL8vFjEEuc6FL2m1amZixVLTwQ5mpXcM,9833
-scdataloader-1.6.3.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-scdataloader-1.6.3.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-scdataloader-1.6.3.dist-info/RECORD,,