scdataloader 1.1.3__py3-none-any.whl → 1.2.2__py3-none-any.whl

scdataloader/utils.py

@@ -1,23 +1,21 @@
 import io
 import os
 import urllib
+from collections import Counter
+from functools import lru_cache
+from typing import List, Optional, Union
 
 import bionty as bt
 import lamindb as ln
 import numpy as np
 import pandas as pd
+import torch
+from anndata import AnnData
 from biomart import BiomartServer
 from django.db import IntegrityError
 from scipy.sparse import csr_matrix
 from scipy.stats import median_abs_deviation
-from functools import lru_cache
-from collections import Counter
 from torch import Tensor
-import torch
-
-from typing import Union, List, Optional
-
-from anndata import AnnData
 
 
 def downsample_profile(mat: Tensor, dropout: float):
@@ -92,7 +90,7 @@ def _fetchFromServer(
 
 
 def getBiomartTable(
-    ensemble_server: str = "http://jul2023.archive.ensembl.org/biomart",
+    ensemble_server: str = "http://may2024.archive.ensembl.org/biomart",
     useCache: bool = False,
     cache_folder: str = "/tmp/biomart/",
     attributes: List[str] = [],
@@ -102,7 +100,7 @@ def getBiomartTable(
     """generate a genelist dataframe from ensembl's biomart
 
     Args:
-        ensemble_server (str, optional): the biomart server. Defaults to "http://jul2023.archive.ensembl.org/biomart".
+        ensemble_server (str, optional): the biomart server. Defaults to "http://may2023.archive.ensembl.org/biomart".
         useCache (bool, optional): whether to use the cache or not. Defaults to False.
         cache_folder (str, optional): the cache folder. Defaults to "/tmp/biomart/".
         attributes (List[str], optional): the attributes to fetch. Defaults to [].
@@ -129,21 +127,20 @@ def getBiomartTable(
 
     cache_folder = os.path.expanduser(cache_folder)
     createFoldersFor(cache_folder)
-    cachefile = os.path.join(cache_folder, ".biomart.csv")
+    cachefile = os.path.join(cache_folder, ".biomart.parquet")
     if useCache & os.path.isfile(cachefile):
         print("fetching gene names from biomart cache")
-        res = pd.read_csv(cachefile)
+        res = pd.read_parquet(cachefile)
     else:
         print("downloading gene names from biomart")
 
         res = _fetchFromServer(ensemble_server, attr + attributes, database=database)
-        res.to_csv(cachefile, index=False)
+        res.to_parquet(cachefile, index=False)
     res.columns = attr + attributes
     if type(res) is not type(pd.DataFrame()):
         raise ValueError("should be a dataframe")
     res = res[~(res["ensembl_gene_id"].isna())]
     if "hgnc_symbol" in res.columns:
-        res = res[res["hgnc_symbol"].isna()]
         res.loc[res[res.hgnc_symbol.isna()].index, "hgnc_symbol"] = res[
             res.hgnc_symbol.isna()
         ]["ensembl_gene_id"]
@@ -371,10 +368,16 @@ def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10
         genesdf["organism"] = organism
         organismdf.append(genesdf)
     organismdf = pd.concat(organismdf)
-    organismdf.drop(
-        columns=["source_id", "run_id", "created_by_id", "updated_at", "stable_id"],
-        inplace=True,
-    )
+    for col in [
+        "source_id",
+        "run_id",
+        "created_by_id",
+        "updated_at",
+        "stable_id",
+        "created_at",
+    ]:
+        if col in organismdf.columns:
+            organismdf.drop(columns=[col], inplace=True)
     return organismdf
 
 
@@ -387,6 +390,7 @@ def populate_my_ontology(
     tissues: List[str] = [],
     diseases: List[str] = [],
     dev_stages: List[str] = [],
+    organism_clade: str = "vertebrates",
 ):
     """
     creates a local version of the lamin ontologies and add the required missing values in base ontologies
@@ -397,7 +401,7 @@ def populate_my_ontology(
 
     add whatever value you need afterward like it is done here with:
 
-    `bt.$ontology(name="ddd", ontology_id="ddddd").save()`
+    `bt.$ontology(name="ddd", ontolbogy_id="ddddd").save()`
 
     `df["assay_ontology_term_id"].unique()`
 
@@ -414,89 +418,111 @@ def populate_my_ontology(
     """
     # cell type
     if celltypes is not None:
-        names = bt.CellType.public().df().index if not celltypes else celltypes
-        records = bt.CellType.from_values(names, field="ontology_id")
-        ln.save(records)
+        if len(celltypes) == 0:
+            bt.CellType.import_from_source(update=True)
+        else:
+            names = bt.CellType.public().df().index if not celltypes else celltypes
+            records = bt.CellType.from_values(names, field="ontology_id")
+            ln.save(records)
         bt.CellType(name="unknown", ontology_id="unknown").save()
     # Organism
     if organisms is not None:
-        names = bt.Organism.public().df().index if not organisms else organisms
+        names = (
+            bt.Organism.public(organism=organism_clade).df().index
+            if not organisms
+            else organisms
+        )
+        source = bt.PublicSource.filter(name="ensembl", organism=organism_clade).last()
         records = [
             i[0] if type(i) is list else i
-            for i in [bt.Organism.from_source(ontology_id=i) for i in names]
+            for i in [
+                bt.Organism.from_source(ontology_id=i, source=source) for i in names
+            ]
         ]
         ln.save(records)
         bt.Organism(name="unknown", ontology_id="unknown").save()
-        organism_names = names
     # Phenotype
     if sex is not None:
         names = bt.Phenotype.public().df().index if not sex else sex
+        source = bt.PublicSource.filter(name="pato").first()
         records = [
-            bt.Phenotype.from_source(
-                ontology_id=i, source=bt.PublicSource.filter(name="pato").first()
-            )
-            for i in names
+            bt.Phenotype.from_source(ontology_id=i, source=source) for i in names
         ]
         ln.save(records)
         bt.Phenotype(name="unknown", ontology_id="unknown").save()
     # ethnicity
     if ethnicities is not None:
-        names = bt.Ethnicity.public().df().index if not ethnicities else ethnicities
-        records = bt.Ethnicity.from_values(names, field="ontology_id")
-        ln.save(records)
+        if len(ethnicities) == 0:
+            bt.Ethnicity.import_from_source(update=True)
+        else:
+            names = bt.Ethnicity.public().df().index if not ethnicities else ethnicities
+            records = bt.Ethnicity.from_values(names, field="ontology_id")
+            ln.save(records)
         bt.Ethnicity(
             name="unknown", ontology_id="unknown"
         ).save()  # multi ethnic will have to get renamed
     # ExperimentalFactor
     if assays is not None:
-        names = bt.ExperimentalFactor.public().df().index if not assays else assays
-        records = bt.ExperimentalFactor.from_values(names, field="ontology_id")
-        ln.save(records)
+        if len(assays) == 0:
+            bt.ExperimentalFactor.import_from_source(update=True)
+        else:
+            names = bt.ExperimentalFactor.public().df().index if not assays else assays
+            records = bt.ExperimentalFactor.from_values(names, field="ontology_id")
+            ln.save(records)
         bt.ExperimentalFactor(name="unknown", ontology_id="unknown").save()
         # lookup = bt.ExperimentalFactor.lookup()
         # lookup.smart_seq_v4.parents.add(lookup.smart_like)
     # Tissue
     if tissues is not None:
-        names = bt.Tissue.public().df().index if not tissues else tissues
-        records = bt.Tissue.from_values(names, field="ontology_id")
-        ln.save(records)
+        if len(tissues) == 0:
+            bt.Tissue.import_from_source(update=True)
+        else:
+            names = bt.Tissue.public().df().index if not tissues else tissues
+            records = bt.Tissue.from_values(names, field="ontology_id")
+            ln.save(records)
         bt.Tissue(name="unknown", ontology_id="unknown").save()
     # DevelopmentalStage
     if dev_stages is not None:
-        names = (
-            bt.DevelopmentalStage.public().df().index if not dev_stages else dev_stages
-        )
-        records = bt.DevelopmentalStage.from_values(names, field="ontology_id")
-        ln.save(records)
+        if len(dev_stages) == 0:
+            bt.DevelopmentalStage.import_from_source(update=True)
+            source = bt.PublicSource.filter(organism="mouse", name="mmusdv").last()
+            bt.DevelopmentalStage.import_from_source(source=source)
+        else:
+            names = (
+                bt.DevelopmentalStage.public().df().index
+                if not dev_stages
+                else dev_stages
+            )
+            records = bt.DevelopmentalStage.from_values(names, field="ontology_id")
+            ln.save(records)
         bt.DevelopmentalStage(name="unknown", ontology_id="unknown").save()
 
-        names = bt.DevelopmentalStage.public(organism="mouse").df().index
-        records = [
-            bt.DevelopmentalStage.from_source(
-                ontology_id=i,
-                source=bt.PublicSource.filter(organism="mouse", name="mmusdv").first(),
-            )
-            for i in names.tolist()
-        ]
-        ln.save(records)
     # Disease
     if diseases is not None:
-        names = bt.Disease.public().df().index if not diseases else diseases
-        records = bt.Disease.from_values(names, field="ontology_id")
-        ln.save(records)
+        if len(diseases) == 0:
+            bt.Disease.import_from_source(update=True)
+        else:
+            names = bt.Disease.public().df().index if not diseases else diseases
+            records = bt.Disease.from_values(names, field="ontology_id")
+            ln.save(records)
         bt.Disease(name="normal", ontology_id="PATO:0000461").save()
         bt.Disease(name="unknown", ontology_id="unknown").save()
     # genes
-    for organism in organism_names:
+    for organism in ["NCBITaxon:10090", "NCBITaxon:9606"]:
         # convert onto to name
         organism = bt.Organism.filter(ontology_id=organism).one().name
         names = bt.Gene.public(organism=organism).df()["ensembl_gene_id"]
-        records = bt.Gene.from_values(
-            names,
-            field="ensembl_gene_id",
-            organism=organism,
-        )
-        ln.save(records)
+
+        # Process names in blocks of 10,000 elements
+        block_size = 10000
+        for i in range(0, len(names), block_size):
+            block = names[i : i + block_size]
+            records = bt.Gene.from_values(
+                block,
+                field="ensembl_gene_id",
+                organism=organism,
+            )
+            ln.save(records)
 
 
 def is_outlier(adata: AnnData, metric: str, nmads: int):

scdataloader-1.2.2.dist-info/METADATA

@@ -0,0 +1,299 @@
+Metadata-Version: 2.3
+Name: scdataloader
+Version: 1.2.2
+Summary: a dataloader for single cell data in lamindb
+Project-URL: repository, https://github.com/jkobject/scDataLoader
+Author-email: jkobject <jkobject@gmail.com>
+License: MIT
+Keywords: dataloader,lamindb,pytorch,scPRINT,scRNAseq
+Requires-Python: <3.11,>=3.10
+Requires-Dist: anndata>=0.9.0
+Requires-Dist: biomart>=0.9.0
+Requires-Dist: cellxgene-census>=0.1.0
+Requires-Dist: django>=4.0.0
+Requires-Dist: ipykernel>=6.20.0
+Requires-Dist: lamindb[bionty]==0.76.12
+Requires-Dist: leidenalg>=0.8.0
+Requires-Dist: lightning>=2.0.0
+Requires-Dist: matplotlib>=3.5.0
+Requires-Dist: numpy>=1.26.0
+Requires-Dist: palantir>=1.3.3
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: scikit-misc>=0.5.0
+Requires-Dist: seaborn>=0.11.0
+Requires-Dist: torch==2.2.0
+Requires-Dist: torchdata>=0.5.0
+Provides-Extra: dev
+Requires-Dist: coverage>=7.3.2; extra == 'dev'
+Requires-Dist: gitchangelog>=3.0.4; extra == 'dev'
+Requires-Dist: mkdocs-git-authors-plugin>=0.4.0; extra == 'dev'
+Requires-Dist: mkdocs-git-revision-date-localized-plugin>=1.0.0; extra == 'dev'
+Requires-Dist: mkdocs-jupyter>=0.2.0; extra == 'dev'
+Requires-Dist: mkdocs>=1.5.3; extra == 'dev'
+Requires-Dist: mkdocstrings-python>=0.10.0; extra == 'dev'
+Requires-Dist: mkdocstrings>=0.22.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
+Requires-Dist: pytest>=7.4.3; extra == 'dev'
+Requires-Dist: ruff>=0.6.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# scdataloader
+
+[![codecov](https://codecov.io/gh/jkobject/scDataLoader/branch/main/graph/badge.svg?token=scDataLoader_token_here)](https://codecov.io/gh/jkobject/scDataLoader)
+[![CI](https://github.com/jkobject/scDataLoader/actions/workflows/main.yml/badge.svg)](https://github.com/jkobject/scDataLoader/actions/workflows/main.yml)
+[![PyPI version](https://badge.fury.io/py/scDataLoader.svg)](https://badge.fury.io/py/scDataLoader)
+[![Downloads](https://pepy.tech/badge/scDataLoader)](https://pepy.tech/project/scDataLoader)
+[![Downloads](https://pepy.tech/badge/scDataLoader/month)](https://pepy.tech/project/scDataLoader)
+[![Downloads](https://pepy.tech/badge/scDataLoader/week)](https://pepy.tech/project/scDataLoader)
+[![GitHub issues](https://img.shields.io/github/issues/jkobject/scDataLoader)](https://img.shields.io/github/issues/jkobject/scDataLoader)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![DOI](https://img.shields.io/badge/DOI-10.1101%2F2024.07.29.605556-blue)](https://doi.org/10.1101/2024.07.29.605556)
+
+This single cell pytorch dataloader / lighting datamodule is designed to be used with:
+
+- [lamindb](https://lamin.ai/)
+
+and:
+
+- [scanpy](https://scanpy.readthedocs.io/en/stable/)
+- [anndata](https://anndata.readthedocs.io/en/latest/)
+
+It allows you to:
+
+1. load thousands of datasets containing millions of cells in a few seconds.
+2. preprocess the data per dataset and download it locally (normalization, filtering, etc.)
+3. create a more complex single cell dataset
+4. extend it to your need
+
+built on top of `lamindb` and the `.mapped()` function by Sergey: https://github.com/Koncopd 
+
+The package has been designed together with the [scPRINT paper](https://doi.org/10.1101/2024.07.29.605556) and [model](https://github.com/cantinilab/scPRINT).
+
+## More
+
+I needed to create this Data Loader for my PhD project. I am using it to load & preprocess thousands of datasets containing millions of cells in a few seconds. I believed that individuals employing AI for single-cell RNA sequencing and other sequencing datasets would eagerly utilize and desire such a tool, which presently does not exist.
+
+![scdataloader.drawio.png](docs/scdataloader.drawio.png)
+
+## Install it from PyPI
+
+```bash
+pip install scdataloader
+# or
+pip install scDataLoader[dev] # for dev dependencies
+
+lamin init --storage ./testdb --name test --schema bionty
+```
+
+if you start with lamin and had to do a `lamin init`, you will also need to populate your ontologies. This is because scPRINT is using ontologies to define its cell types, diseases, sexes, ethnicities, etc.
+
+you can do it manually or with our function:
+
+```python
+from scdataloader.utils import populate_my_ontology
+
+populate_my_ontology() #to populate everything (recommended) (can take 2-10mns)
+
+populate_my_ontology( #the minimum to the tool
+organisms: List[str] = ["NCBITaxon:10090", "NCBITaxon:9606"],
+    sex: List[str] = ["PATO:0000384", "PATO:0000383"],
+    celltypes = None,
+    ethnicities = None,
+    assays = None,
+    tissues = None,
+    diseases = None,
+    dev_stages = None,
+)
+```
+
+### Dev install
+
+If you want to use the latest version of scDataLoader and work on the code yourself use `git clone` and `pip -e` instead of `pip install`.
+
+```bash
+git clone https://github.com/jkobject/scDataLoader.git
+pip install -e scDataLoader[dev]
+```
+
+## Usage
+
+### DataModule usage
+
+```python
+# initialize a local lamin database
+#! lamin init --storage ./cellxgene --name cellxgene --schema bionty
+from scdataloader import utils, Preprocessor, DataModule
+
+
+# preprocess datasets
+preprocessor = Preprocessor(
+    do_postp=False,
+    force_preprocess=True,
+)
+adata = preprocessor(adata)
+
+art = ln.Artifact(adata, description="test")
+art.save()
+ln.Collection(art, name="test", description="test").save()
+
+datamodule = DataModule(
+    collection_name="test",
+    organisms=["NCBITaxon:9606"], #organism that we will work on
+    how="most expr", # for the collator (most expr genes only will be selected)
+    max_len=1000, # only the 1000 most expressed
+    batch_size=64,
+    num_workers=1,
+    validation_split=0.1,
+)
+```
+
+### lightning-free usage (Dataset+Collator+DataLoader)
+
+```python
+# initialize a local lamin database
+#! lamin init --storage ./cellxgene --name cellxgene --schema bionty
+
+from scdataloader import utils, Preprocessor, SimpleAnnDataset, Collator, DataLoader
+
+# preprocess dataset
+preprocessor = Preprocessor(
+    do_postp=False,
+    force_preprocess=True,
+)
+adata = preprocessor(adata)
+
+# create dataset
+adataset = SimpleAnnDataset(
+    adata, obs_to_output=["organism_ontology_term_id"]
+)
+# create collator
+col = Collator(
+    organisms="NCBITaxon:9606",
+    valid_genes=adata.var_names,
+    max_len=2000, #maximum number of genes to use
+    how="some" |"most expr"|"random_expr",
+    # genelist = [geneA, geneB] if how=='some'
+)
+# create dataloader
+dataloader = DataLoader(
+    adataset,
+    collate_fn=col,
+    batch_size=64,
+    num_workers=4,
+    shuffle=False,
+)
+
+# predict
+for batch in tqdm(dataloader):
+    gene_pos, expression, depth = (
+        batch["genes"],
+        batch["x"],
+        batch["depth"],
+    )
+    model.predict(
+        gene_pos,
+        expression,
+        depth,
+    )
+```
+
+### Usage on all of cellxgene
+
+```python
+# initialize a local lamin database
+#! lamin init --storage ./cellxgene --name cellxgene --schema bionty
+
+from scdataloader import utils
+from scdataloader.preprocess import LaminPreprocessor, additional_postprocess, additional_preprocess
+
+# preprocess datasets
+DESCRIPTION='preprocessed by scDataLoader'
+
+cx_dataset = ln.Collection.using(instance="laminlabs/cellxgene").filter(name="cellxgene-census", version='2023-12-15').one()
+cx_dataset, len(cx_dataset.artifacts.all())
+
+
+do_preprocess = LaminPreprocessor(additional_postprocess=additional_postprocess, additional_preprocess=additional_preprocess, skip_validate=True, subset_hvg=0)
+
+preprocessed_dataset = do_preprocess(cx_dataset, name=DESCRIPTION, description=DESCRIPTION, start_at=6, version="2")
+
+# create dataloaders
+from scdataloader import DataModule
+import tqdm
+
+datamodule = DataModule(
+    collection_name="preprocessed dataset",
+    organisms=["NCBITaxon:9606"], #organism that we will work on
+    how="most expr", # for the collator (most expr genes only will be selected)
+    max_len=1000, # only the 1000 most expressed
+    batch_size=64,
+    num_workers=1,
+    validation_split=0.1,
+    test_split=0)
+
+for i in tqdm.tqdm(datamodule.train_dataloader()):
+    # pass #or do pass
+    print(i)
+    break
+
+# with lightning:
+# Trainer(model, datamodule)
+
+```
+
+see the notebooks in [docs](https://www.jkobject.com/scDataLoader/):
+
+1. [load a dataset](https://www.jkobject.com/scDataLoader/notebooks/1_download_and_preprocess/)
+2. [create a dataset](https://www.jkobject.com/scDataLoader/notebooks/2_create_dataloader/)
+
+### command line preprocessing
+
+You can use the command line to preprocess a large database of datasets like here for cellxgene. this allows parallelizing and easier usage.
+
+```bash
+scdataloader --instance "laminlabs/cellxgene" --name "cellxgene-census" --version "2023-12-15" --description "preprocessed for scprint" --new_name "scprint main" --start_at 10 >> scdataloader.out
+```
+
+### command line usage
+
+The main way to use
+
+> please refer to the [scPRINT documentation](https://www.jkobject.com/scPRINT/) and [lightning documentation](https://lightning.ai/docs/pytorch/stable/cli/lightning_cli_intermediate.html) for more information on command line usage
+
+## FAQ
+
+### how to update my ontologies?
+
+```bash
+import bionty as bt
+bt.reset_sources()
+
+# Run via CLI: lamin load <your instance>
+
+import lnschema_bionty as lb
+lb.dev.sync_bionty_source_to_latest()
+```
+
+### how to load all ontologies?
+
+```python
+from scdataloader import utils
+utils.populate_ontologies() # this might take from 5-20mins
+```
+
+## Development
+
+Read the [CONTRIBUTING.md](CONTRIBUTING.md) file.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Acknowledgments
+
+- [lamin.ai](https://lamin.ai/)
+- [scanpy](https://scanpy.readthedocs.io/en/stable/)
+- [anndata](https://anndata.readthedocs.io/en/latest/)
+- [scprint](https://www.jkobject.com/scPRINT/)
+
+Awesome single cell dataloader created by @jkobject

scdataloader-1.2.2.dist-info/RECORD

@@ -0,0 +1,14 @@
+scdataloader/VERSION,sha256=xipcxhrEUlk1dT9ewoTAoFKksdpLOjWA3OK313ohVK4,6
+scdataloader/__init__.py,sha256=5y9VzRhOAUWeYMn2MrRRRlzgdiMjRFytr7gcn-I6IkE,147
+scdataloader/__main__.py,sha256=VXrt2IykBypnIXWydwA7NfF7LtRGc-0Khjtm5OIBNpI,6527
+scdataloader/base.py,sha256=M1gD59OffRdLOgS1vHKygOomUoAMuzjpRtAfM3SBKF8,338
+scdataloader/collator.py,sha256=gzHiuixUwK8JClhAbG12kgWMU_VTKkowibA-tDFpbwo,11341
+scdataloader/config.py,sha256=rrW2DZxG4J2_pmpDbXXsaKJkpNC57w5dIlItiFbANYw,2905
+scdataloader/data.py,sha256=3dCp-lIAfOkCi76SH5W3iSqFmAWZslwARkN9v5mylz8,14907
+scdataloader/datamodule.py,sha256=B-udBevPSPF__hfy0pOz1dGovgE95K2pxPupjB7RblI,16936
+scdataloader/preprocess.py,sha256=pH4EPrcRqH34o3t5X3A4kETiYdCZngih5SdP_PPfgOo,29178
+scdataloader/utils.py,sha256=7tgt3sPj_XTKb-UlJDAZWvQr0_DG9VTC6ioiLdBWFFE,22498
+scdataloader-1.2.2.dist-info/METADATA,sha256=XMtKO9ImiyY--F92njvMUe69OaJgDx8C3xQtBAXqo8g,9800
+scdataloader-1.2.2.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+scdataloader-1.2.2.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+scdataloader-1.2.2.dist-info/RECORD,,

{scdataloader-1.1.3.dist-info → scdataloader-1.2.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: hatchling 1.26.3
 Root-Is-Purelib: true
 Tag: py3-none-any