scdataloader 1.1.3__tar.gz → 1.2.2__tar.gz

scdataloader-1.2.2/.gitignore

@@ -0,0 +1,133 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# templates
+.github/templates/*
+.DS_Store

scdataloader-1.2.2/PKG-INFO

@@ -0,0 +1,299 @@
+Metadata-Version: 2.3
+Name: scdataloader
+Version: 1.2.2
+Summary: a dataloader for single cell data in lamindb
+Project-URL: repository, https://github.com/jkobject/scDataLoader
+Author-email: jkobject <jkobject@gmail.com>
+License: MIT
+Keywords: dataloader,lamindb,pytorch,scPRINT,scRNAseq
+Requires-Python: <3.11,>=3.10
+Requires-Dist: anndata>=0.9.0
+Requires-Dist: biomart>=0.9.0
+Requires-Dist: cellxgene-census>=0.1.0
+Requires-Dist: django>=4.0.0
+Requires-Dist: ipykernel>=6.20.0
+Requires-Dist: lamindb[bionty]==0.76.12
+Requires-Dist: leidenalg>=0.8.0
+Requires-Dist: lightning>=2.0.0
+Requires-Dist: matplotlib>=3.5.0
+Requires-Dist: numpy>=1.26.0
+Requires-Dist: palantir>=1.3.3
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: scikit-misc>=0.5.0
+Requires-Dist: seaborn>=0.11.0
+Requires-Dist: torch==2.2.0
+Requires-Dist: torchdata>=0.5.0
+Provides-Extra: dev
+Requires-Dist: coverage>=7.3.2; extra == 'dev'
+Requires-Dist: gitchangelog>=3.0.4; extra == 'dev'
+Requires-Dist: mkdocs-git-authors-plugin>=0.4.0; extra == 'dev'
+Requires-Dist: mkdocs-git-revision-date-localized-plugin>=1.0.0; extra == 'dev'
+Requires-Dist: mkdocs-jupyter>=0.2.0; extra == 'dev'
+Requires-Dist: mkdocs>=1.5.3; extra == 'dev'
+Requires-Dist: mkdocstrings-python>=0.10.0; extra == 'dev'
+Requires-Dist: mkdocstrings>=0.22.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
+Requires-Dist: pytest>=7.4.3; extra == 'dev'
+Requires-Dist: ruff>=0.6.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# scdataloader
+
+[![codecov](https://codecov.io/gh/jkobject/scDataLoader/branch/main/graph/badge.svg?token=scDataLoader_token_here)](https://codecov.io/gh/jkobject/scDataLoader)
+[![CI](https://github.com/jkobject/scDataLoader/actions/workflows/main.yml/badge.svg)](https://github.com/jkobject/scDataLoader/actions/workflows/main.yml)
+[![PyPI version](https://badge.fury.io/py/scDataLoader.svg)](https://badge.fury.io/py/scDataLoader)
+[![Downloads](https://pepy.tech/badge/scDataLoader)](https://pepy.tech/project/scDataLoader)
+[![Downloads](https://pepy.tech/badge/scDataLoader/month)](https://pepy.tech/project/scDataLoader)
+[![Downloads](https://pepy.tech/badge/scDataLoader/week)](https://pepy.tech/project/scDataLoader)
+[![GitHub issues](https://img.shields.io/github/issues/jkobject/scDataLoader)](https://img.shields.io/github/issues/jkobject/scDataLoader)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![DOI](https://img.shields.io/badge/DOI-10.1101%2F2024.07.29.605556-blue)](https://doi.org/10.1101/2024.07.29.605556)
+
+This single cell pytorch dataloader / lighting datamodule is designed to be used with:
+
+- [lamindb](https://lamin.ai/)
+
+and:
+
+- [scanpy](https://scanpy.readthedocs.io/en/stable/)
+- [anndata](https://anndata.readthedocs.io/en/latest/)
+
+It allows you to:
+
+1. load thousands of datasets containing millions of cells in a few seconds.
+2. preprocess the data per dataset and download it locally (normalization, filtering, etc.)
+3. create a more complex single cell dataset
+4. extend it to your need
+
+built on top of `lamindb` and the `.mapped()` function by Sergey: https://github.com/Koncopd 
+
+The package has been designed together with the [scPRINT paper](https://doi.org/10.1101/2024.07.29.605556) and [model](https://github.com/cantinilab/scPRINT).
+
+## More
+
+I needed to create this Data Loader for my PhD project. I am using it to load & preprocess thousands of datasets containing millions of cells in a few seconds. I believed that individuals employing AI for single-cell RNA sequencing and other sequencing datasets would eagerly utilize and desire such a tool, which presently does not exist.
+
+![scdataloader.drawio.png](docs/scdataloader.drawio.png)
+
+## Install it from PyPI
+
+```bash
+pip install scdataloader
+# or
+pip install scDataLoader[dev] # for dev dependencies
+
+lamin init --storage ./testdb --name test --schema bionty
+```
+
+if you start with lamin and had to do a `lamin init`, you will also need to populate your ontologies. This is because scPRINT is using ontologies to define its cell types, diseases, sexes, ethnicities, etc.
+
+you can do it manually or with our function:
+
+```python
+from scdataloader.utils import populate_my_ontology
+
+populate_my_ontology() #to populate everything (recommended) (can take 2-10mns)
+
+populate_my_ontology( #the minimum to the tool
+organisms: List[str] = ["NCBITaxon:10090", "NCBITaxon:9606"],
+    sex: List[str] = ["PATO:0000384", "PATO:0000383"],
+    celltypes = None,
+    ethnicities = None,
+    assays = None,
+    tissues = None,
+    diseases = None,
+    dev_stages = None,
+)
+```
+
+### Dev install
+
+If you want to use the latest version of scDataLoader and work on the code yourself use `git clone` and `pip -e` instead of `pip install`.
+
+```bash
+git clone https://github.com/jkobject/scDataLoader.git
+pip install -e scDataLoader[dev]
+```
+
+## Usage
+
+### DataModule usage
+
+```python
+# initialize a local lamin database
+#! lamin init --storage ./cellxgene --name cellxgene --schema bionty
+from scdataloader import utils, Preprocessor, DataModule
+
+
+# preprocess datasets
+preprocessor = Preprocessor(
+    do_postp=False,
+    force_preprocess=True,
+)
+adata = preprocessor(adata)
+
+art = ln.Artifact(adata, description="test")
+art.save()
+ln.Collection(art, name="test", description="test").save()
+
+datamodule = DataModule(
+    collection_name="test",
+    organisms=["NCBITaxon:9606"], #organism that we will work on
+    how="most expr", # for the collator (most expr genes only will be selected)
+    max_len=1000, # only the 1000 most expressed
+    batch_size=64,
+    num_workers=1,
+    validation_split=0.1,
+)
+```
+
+### lightning-free usage (Dataset+Collator+DataLoader)
+
+```python
+# initialize a local lamin database
+#! lamin init --storage ./cellxgene --name cellxgene --schema bionty
+
+from scdataloader import utils, Preprocessor, SimpleAnnDataset, Collator, DataLoader
+
+# preprocess dataset
+preprocessor = Preprocessor(
+    do_postp=False,
+    force_preprocess=True,
+)
+adata = preprocessor(adata)
+
+# create dataset
+adataset = SimpleAnnDataset(
+    adata, obs_to_output=["organism_ontology_term_id"]
+)
+# create collator
+col = Collator(
+    organisms="NCBITaxon:9606",
+    valid_genes=adata.var_names,
+    max_len=2000, #maximum number of genes to use
+    how="some" |"most expr"|"random_expr",
+    # genelist = [geneA, geneB] if how=='some'
+)
+# create dataloader
+dataloader = DataLoader(
+    adataset,
+    collate_fn=col,
+    batch_size=64,
+    num_workers=4,
+    shuffle=False,
+)
+
+# predict
+for batch in tqdm(dataloader):
+    gene_pos, expression, depth = (
+        batch["genes"],
+        batch["x"],
+        batch["depth"],
+    )
+    model.predict(
+        gene_pos,
+        expression,
+        depth,
+    )
+```
+
+### Usage on all of cellxgene
+
+```python
+# initialize a local lamin database
+#! lamin init --storage ./cellxgene --name cellxgene --schema bionty
+
+from scdataloader import utils
+from scdataloader.preprocess import LaminPreprocessor, additional_postprocess, additional_preprocess
+
+# preprocess datasets
+DESCRIPTION='preprocessed by scDataLoader'
+
+cx_dataset = ln.Collection.using(instance="laminlabs/cellxgene").filter(name="cellxgene-census", version='2023-12-15').one()
+cx_dataset, len(cx_dataset.artifacts.all())
+
+
+do_preprocess = LaminPreprocessor(additional_postprocess=additional_postprocess, additional_preprocess=additional_preprocess, skip_validate=True, subset_hvg=0)
+
+preprocessed_dataset = do_preprocess(cx_dataset, name=DESCRIPTION, description=DESCRIPTION, start_at=6, version="2")
+
+# create dataloaders
+from scdataloader import DataModule
+import tqdm
+
+datamodule = DataModule(
+    collection_name="preprocessed dataset",
+    organisms=["NCBITaxon:9606"], #organism that we will work on
+    how="most expr", # for the collator (most expr genes only will be selected)
+    max_len=1000, # only the 1000 most expressed
+    batch_size=64,
+    num_workers=1,
+    validation_split=0.1,
+    test_split=0)
+
+for i in tqdm.tqdm(datamodule.train_dataloader()):
+    # pass #or do pass
+    print(i)
+    break
+
+# with lightning:
+# Trainer(model, datamodule)
+
+```
+
+see the notebooks in [docs](https://www.jkobject.com/scDataLoader/):
+
+1. [load a dataset](https://www.jkobject.com/scDataLoader/notebooks/1_download_and_preprocess/)
+2. [create a dataset](https://www.jkobject.com/scDataLoader/notebooks/2_create_dataloader/)
+
+### command line preprocessing
+
+You can use the command line to preprocess a large database of datasets like here for cellxgene. this allows parallelizing and easier usage.
+
+```bash
+scdataloader --instance "laminlabs/cellxgene" --name "cellxgene-census" --version "2023-12-15" --description "preprocessed for scprint" --new_name "scprint main" --start_at 10 >> scdataloader.out
+```
+
+### command line usage
+
+The main way to use
+
+> please refer to the [scPRINT documentation](https://www.jkobject.com/scPRINT/) and [lightning documentation](https://lightning.ai/docs/pytorch/stable/cli/lightning_cli_intermediate.html) for more information on command line usage
+
+## FAQ
+
+### how to update my ontologies?
+
+```bash
+import bionty as bt
+bt.reset_sources()
+
+# Run via CLI: lamin load <your instance>
+
+import lnschema_bionty as lb
+lb.dev.sync_bionty_source_to_latest()
+```
+
+### how to load all ontologies?
+
+```python
+from scdataloader import utils
+utils.populate_ontologies() # this might take from 5-20mins
+```
+
+## Development
+
+Read the [CONTRIBUTING.md](CONTRIBUTING.md) file.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Acknowledgments
+
+- [lamin.ai](https://lamin.ai/)
+- [scanpy](https://scanpy.readthedocs.io/en/stable/)
+- [anndata](https://anndata.readthedocs.io/en/latest/)
+- [scprint](https://www.jkobject.com/scPRINT/)
+
+Awesome single cell dataloader created by @jkobject

{scdataloader-1.1.3 → scdataloader-1.2.2}/README.md

@@ -3,7 +3,6 @@
 [![codecov](https://codecov.io/gh/jkobject/scDataLoader/branch/main/graph/badge.svg?token=scDataLoader_token_here)](https://codecov.io/gh/jkobject/scDataLoader)
 [![CI](https://github.com/jkobject/scDataLoader/actions/workflows/main.yml/badge.svg)](https://github.com/jkobject/scDataLoader/actions/workflows/main.yml)
 [![PyPI version](https://badge.fury.io/py/scDataLoader.svg)](https://badge.fury.io/py/scDataLoader)
-[![Documentation Status](https://readthedocs.org/projects/scDataLoader/badge/?version=latest)](https://scDataLoader.readthedocs.io/en/latest/?badge=latest)
 [![Downloads](https://pepy.tech/badge/scDataLoader)](https://pepy.tech/project/scDataLoader)
 [![Downloads](https://pepy.tech/badge/scDataLoader/month)](https://pepy.tech/project/scDataLoader)
 [![Downloads](https://pepy.tech/badge/scDataLoader/week)](https://pepy.tech/project/scDataLoader)
@@ -44,8 +43,7 @@ pip install scdataloader
 # or
 pip install scDataLoader[dev] # for dev dependencies
 
-lamin login <email> --key <API-key>
-lamin init --storage [folder-name-where-lamin-data-will-be-stored] --schema bionty
+lamin init --storage ./testdb --name test --schema bionty
 ```
 
 if you start with lamin and had to do a `lamin init`, you will also need to populate your ontologies. This is because scPRINT is using ontologies to define its cell types, diseases, sexes, ethnicities, etc.
@@ -57,7 +55,7 @@ from scdataloader.utils import populate_my_ontology
 
 populate_my_ontology() #to populate everything (recommended) (can take 2-10mns)
 
-populate_my_ontology( #the minimum for scprint to run some inferences (denoising, grn inference)
+populate_my_ontology( #the minimum to the tool
 organisms: List[str] = ["NCBITaxon:10090", "NCBITaxon:9606"],
     sex: List[str] = ["PATO:0000384", "PATO:0000383"],
     celltypes = None,
@@ -80,11 +78,91 @@ pip install -e scDataLoader[dev]
 
 ## Usage
 
-### Direct Usage
+### DataModule usage
 
 ```python
 # initialize a local lamin database
-# !lamin init --storage ~/scdataloader --schema bionty
+#! lamin init --storage ./cellxgene --name cellxgene --schema bionty
+from scdataloader import utils, Preprocessor, DataModule
+
+
+# preprocess datasets
+preprocessor = Preprocessor(
+    do_postp=False,
+    force_preprocess=True,
+)
+adata = preprocessor(adata)
+
+art = ln.Artifact(adata, description="test")
+art.save()
+ln.Collection(art, name="test", description="test").save()
+
+datamodule = DataModule(
+    collection_name="test",
+    organisms=["NCBITaxon:9606"], #organism that we will work on
+    how="most expr", # for the collator (most expr genes only will be selected)
+    max_len=1000, # only the 1000 most expressed
+    batch_size=64,
+    num_workers=1,
+    validation_split=0.1,
+)
+```
+
+### lightning-free usage (Dataset+Collator+DataLoader)
+
+```python
+# initialize a local lamin database
+#! lamin init --storage ./cellxgene --name cellxgene --schema bionty
+
+from scdataloader import utils, Preprocessor, SimpleAnnDataset, Collator, DataLoader
+
+# preprocess dataset
+preprocessor = Preprocessor(
+    do_postp=False,
+    force_preprocess=True,
+)
+adata = preprocessor(adata)
+
+# create dataset
+adataset = SimpleAnnDataset(
+    adata, obs_to_output=["organism_ontology_term_id"]
+)
+# create collator
+col = Collator(
+    organisms="NCBITaxon:9606",
+    valid_genes=adata.var_names,
+    max_len=2000, #maximum number of genes to use
+    how="some" |"most expr"|"random_expr",
+    # genelist = [geneA, geneB] if how=='some'
+)
+# create dataloader
+dataloader = DataLoader(
+    adataset,
+    collate_fn=col,
+    batch_size=64,
+    num_workers=4,
+    shuffle=False,
+)
+
+# predict
+for batch in tqdm(dataloader):
+    gene_pos, expression, depth = (
+        batch["genes"],
+        batch["x"],
+        batch["depth"],
+    )
+    model.predict(
+        gene_pos,
+        expression,
+        depth,
+    )
+```
+
+### Usage on all of cellxgene
+
+```python
+# initialize a local lamin database
+#! lamin init --storage ./cellxgene --name cellxgene --schema bionty
 
 from scdataloader import utils
 from scdataloader.preprocess import LaminPreprocessor, additional_postprocess, additional_preprocess

scdataloader-1.2.2/pyproject.toml

@@ -0,0 +1,78 @@
+[project]
+name = "scdataloader"
+version = "1.2.2"
+description = "a dataloader for single cell data in lamindb"
+authors = [
+    {name = "jkobject", email = "jkobject@gmail.com"}
+]
+license = "MIT"
+readme = "README.md"
+requires-python = ">=3.10,<3.11"
+keywords = ["scRNAseq", "dataloader", "pytorch", "lamindb", "scPRINT"]
+dependencies = [
+    "numpy>=1.26.0",
+    "lamindb[bionty]==0.76.12",
+    "cellxgene-census>=0.1.0",
+    "torch==2.2.0",
+    "lightning>=2.0.0",
+    "anndata>=0.9.0",
+    "matplotlib>=3.5.0",
+    "seaborn>=0.11.0",
+    "ipykernel>=6.20.0",
+    "torchdata>=0.5.0",
+    "biomart>=0.9.0",
+    "pandas>=2.0.0",
+    "leidenalg>=0.8.0",
+    "django>=4.0.0",
+    "scikit-misc>=0.5.0",
+    "palantir>=1.3.3",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.4.3",
+    "coverage>=7.3.2",
+    "pytest-cov>=4.1.0",
+    "ruff>=0.6.4",
+    "gitchangelog>=3.0.4",
+    "mkdocs>=1.5.3",
+    "mkdocs-git-revision-date-localized-plugin>=1.0.0",
+    "mkdocstrings>=0.22.0",
+    "mkdocs-git-authors-plugin>=0.4.0",
+    "mkdocs-jupyter>=0.2.0",
+    "mkdocstrings-python>=0.10.0"
+]
+
+[tool.ruff]
+# Set the maximum line length to 88.
+line-length = 88
+
+[tool.ruff.lint]
+select = ["E", "F", "I"]
+ignore = ["E501", "E203", "E266", "E265", "F401", "F403", "E722", "E741", "E731", "E721"]
+
+[project.urls]
+repository = "https://github.com/jkobject/scDataLoader"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.sdist]
+only-include = [
+    "/scdataloader",
+]
+
+[tool.hatch.build.targets.wheel]
+only-include = [
+    "/scdataloader",
+]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.uv.sources]
+scdataloader = { workspace = true }
+
+[scripts]
+scdataloader = 'scdataloader.__main__:main'

scdataloader-1.2.2/scdataloader/VERSION

@@ -0,0 +1 @@
+1.2.2

{scdataloader-1.1.3 → scdataloader-1.2.2}/scdataloader/__main__.py

@@ -1,11 +1,13 @@
 import argparse
+from typing import Optional, Union
+
+import lamindb as ln
+
 from scdataloader.preprocess import (
     LaminPreprocessor,
-    additional_preprocess,
     additional_postprocess,
+    additional_preprocess,
 )
-import lamindb as ln
-from typing import Optional, Union
 
 
 # scdataloader --instance="laminlabs/cellxgene" --name="cellxgene-census" --version="2023-12-15" --description="preprocessed for scprint" --new_name="scprint main" --start_at=39
@@ -51,14 +53,14 @@ def main():
     )
     parser.add_argument(
         "--filter_gene_by_counts",
-        type=Union[int, bool],
-        default=False,
+        type=int,
+        default=0,
         help="Determines whether to filter genes by counts.",
     )
     parser.add_argument(
         "--filter_cell_by_counts",
-        type=Union[int, bool],
-        default=False,
+        type=int,
+        default=0,
         help="Determines whether to filter cells by counts.",
     )
     parser.add_argument(
@@ -151,6 +153,12 @@ def main():
         default=False,
         help="Determines whether to do postprocessing.",
     )
+    parser.add_argument(
+        "--cache",
+        type=bool,
+        default=True,
+        help="Determines whether to cache the dataset.",
+    )
     args = parser.parse_args()
 
     # Load the collection
@@ -176,6 +184,7 @@ def main():
         normalize_sum=args.normalize_sum,
         subset_hvg=args.subset_hvg,
         hvg_flavor=args.hvg_flavor,
+        cache=args.cache,
         binning=args.binning,
         result_binned_key=args.result_binned_key,
         length_normalize=args.length_normalize,

{scdataloader-1.1.3 → scdataloader-1.2.2}/scdataloader/collator.py

@@ -1,7 +1,9 @@
+from typing import Optional
+
 import numpy as np
-from .utils import load_genes, downsample_profile
 from torch import Tensor, long
-from typing import Optional
+
+from .utils import downsample_profile, load_genes
 
 
 class Collator: