scdataloader 1.2.1__tar.gz → 1.6.3__tar.gz

{scdataloader-1.2.1 → scdataloader-1.6.3}/PKG-INFO

@@ -1,23 +1,24 @@
 Metadata-Version: 2.3
 Name: scdataloader
-Version: 1.2.1
+Version: 1.6.3
 Summary: a dataloader for single cell data in lamindb
 Project-URL: repository, https://github.com/jkobject/scDataLoader
 Author-email: jkobject <jkobject@gmail.com>
-License-Expression: MIT
-License-File: LICENSE
+License: MIT
 Keywords: dataloader,lamindb,pytorch,scPRINT,scRNAseq
 Requires-Python: <3.11,>=3.10
 Requires-Dist: anndata>=0.9.0
 Requires-Dist: biomart>=0.9.0
 Requires-Dist: cellxgene-census>=0.1.0
 Requires-Dist: django>=4.0.0
+Requires-Dist: harmonypy>=0.0.10
 Requires-Dist: ipykernel>=6.20.0
 Requires-Dist: lamindb[bionty]==0.76.12
 Requires-Dist: leidenalg>=0.8.0
 Requires-Dist: lightning>=2.0.0
 Requires-Dist: matplotlib>=3.5.0
 Requires-Dist: numpy>=1.26.0
+Requires-Dist: palantir>=1.3.3
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: scikit-misc>=0.5.0
 Requires-Dist: seaborn>=0.11.0

{scdataloader-1.2.1 → scdataloader-1.6.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scdataloader"
-version = "1.2.1"
+version = "1.6.3"
 description = "a dataloader for single cell data in lamindb"
 authors = [
     {name = "jkobject", email = "jkobject@gmail.com"}
@@ -25,6 +25,8 @@ dependencies = [
     "leidenalg>=0.8.0",
     "django>=4.0.0",
     "scikit-misc>=0.5.0",
+    "palantir>=1.3.3",
+    "harmonypy>=0.0.10",
 ]
 
 [project.optional-dependencies]
@@ -57,6 +59,16 @@ repository = "https://github.com/jkobject/scDataLoader"
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
+[tool.hatch.build.targets.sdist]
+only-include = [
+    "/scdataloader",
+]
+
+[tool.hatch.build.targets.wheel]
+only-include = [
+    "/scdataloader",
+]
+
 [tool.hatch.metadata]
 allow-direct-references = true
 

scdataloader-1.6.3/scdataloader/VERSION

@@ -0,0 +1 @@
+1.6.3

{scdataloader-1.2.1 → scdataloader-1.6.3}/scdataloader/__main__.py

@@ -53,14 +53,14 @@ def main():
     )
     parser.add_argument(
         "--filter_gene_by_counts",
-        type=Union[int, bool],
-        default=False,
+        type=int,
+        default=0,
         help="Determines whether to filter genes by counts.",
     )
     parser.add_argument(
         "--filter_cell_by_counts",
-        type=Union[int, bool],
-        default=False,
+        type=int,
+        default=0,
         help="Determines whether to filter cells by counts.",
     )
     parser.add_argument(
@@ -153,6 +153,12 @@ def main():
         default=False,
         help="Determines whether to do postprocessing.",
     )
+    parser.add_argument(
+        "--cache",
+        type=bool,
+        default=True,
+        help="Determines whether to cache the dataset.",
+    )
     args = parser.parse_args()
 
     # Load the collection
@@ -178,6 +184,7 @@ def main():
         normalize_sum=args.normalize_sum,
         subset_hvg=args.subset_hvg,
         hvg_flavor=args.hvg_flavor,
+        cache=args.cache,
         binning=args.binning,
         result_binned_key=args.result_binned_key,
         length_normalize=args.length_normalize,

{scdataloader-1.2.1 → scdataloader-1.6.3}/scdataloader/utils.py

@@ -127,15 +127,15 @@ def getBiomartTable(
 
     cache_folder = os.path.expanduser(cache_folder)
     createFoldersFor(cache_folder)
-    cachefile = os.path.join(cache_folder, ".biomart.csv")
+    cachefile = os.path.join(cache_folder, ".biomart.parquet")
     if useCache & os.path.isfile(cachefile):
         print("fetching gene names from biomart cache")
-        res = pd.read_csv(cachefile)
+        res = pd.read_parquet(cachefile)
     else:
         print("downloading gene names from biomart")
 
         res = _fetchFromServer(ensemble_server, attr + attributes, database=database)
-        res.to_csv(cachefile, index=False)
+        res.to_parquet(cachefile, index=False)
     res.columns = attr + attributes
     if type(res) is not type(pd.DataFrame()):
         raise ValueError("should be a dataframe")
@@ -368,7 +368,14 @@ def load_genes(organisms: Union[str, list] = "NCBITaxon:9606"):  # "NCBITaxon:10
         genesdf["organism"] = organism
         organismdf.append(genesdf)
     organismdf = pd.concat(organismdf)
-    for col in ["source_id", "run_id", "created_by_id", "updated_at", "stable_id", "created_at"]:
+    for col in [
+        "source_id",
+        "run_id",
+        "created_by_id",
+        "updated_at",
+        "stable_id",
+        "created_at",
+    ]:
         if col in organismdf.columns:
             organismdf.drop(columns=[col], inplace=True)
     return organismdf
@@ -412,7 +419,7 @@ def populate_my_ontology(
     # cell type
     if celltypes is not None:
         if len(celltypes) == 0:
-            bt.CellType.import_from_source()
+            bt.CellType.import_from_source(update=True)
         else:
             names = bt.CellType.public().df().index if not celltypes else celltypes
             records = bt.CellType.from_values(names, field="ontology_id")
@@ -446,7 +453,7 @@ def populate_my_ontology(
     # ethnicity
     if ethnicities is not None:
         if len(ethnicities) == 0:
-            bt.Ethnicity.import_from_source()
+            bt.Ethnicity.import_from_source(update=True)
         else:
             names = bt.Ethnicity.public().df().index if not ethnicities else ethnicities
             records = bt.Ethnicity.from_values(names, field="ontology_id")
@@ -457,7 +464,7 @@ def populate_my_ontology(
     # ExperimentalFactor
     if assays is not None:
         if len(assays) == 0:
-            bt.ExperimentalFactor.import_from_source()
+            bt.ExperimentalFactor.import_from_source(update=True)
         else:
             names = bt.ExperimentalFactor.public().df().index if not assays else assays
             records = bt.ExperimentalFactor.from_values(names, field="ontology_id")
@@ -468,7 +475,7 @@ def populate_my_ontology(
     # Tissue
     if tissues is not None:
         if len(tissues) == 0:
-            bt.Tissue.import_from_source()
+            bt.Tissue.import_from_source(update=True)
         else:
             names = bt.Tissue.public().df().index if not tissues else tissues
             records = bt.Tissue.from_values(names, field="ontology_id")
@@ -477,7 +484,7 @@ def populate_my_ontology(
     # DevelopmentalStage
     if dev_stages is not None:
         if len(dev_stages) == 0:
-            bt.DevelopmentalStage.import_from_source()
+            bt.DevelopmentalStage.import_from_source(update=True)
             source = bt.PublicSource.filter(organism="mouse", name="mmusdv").last()
             bt.DevelopmentalStage.import_from_source(source=source)
         else:
@@ -493,7 +500,7 @@ def populate_my_ontology(
     # Disease
     if diseases is not None:
         if len(diseases) == 0:
-            bt.Disease.import_from_source()
+            bt.Disease.import_from_source(update=True)
         else:
             names = bt.Disease.public().df().index if not diseases else diseases
             records = bt.Disease.from_values(names, field="ontology_id")

scdataloader-1.2.1/.cursorignore

@@ -1,21 +0,0 @@
-*.json
-*.csv
-*.txt
-*.md
-*.ipynb
-*.Rmd
-*.Rproj
-*.Rproj.user
-*.parquet
-*.ckpt
-*.h5ad
-site/
-*.lamindb
-*.lndb
-*.out
-*.html
-*.pdf
-*.npy
-*.npz
-*.whl
-*.gz

scdataloader-1.2.1/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,31 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: bug, help wanted
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Version [e.g. 22]
-
-**Additional context**
-Add any other context about the problem here.

scdataloader-1.2.1/.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: enhancement, question
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

scdataloader-1.2.1/.github/PULL_REQUEST_TEMPLATE.md

@@ -1,15 +0,0 @@
-### Summary :memo:
-_Write an overview about it._
-
-### Details
-_Describe more what you did on changes._
-1. (...)
-2. (...)
-
-### Bugfixes :bug: (delete if dind't have any)
--
-
-### Checks
-- [ ] Closed #798
-- [ ] Tested Changes
-- [ ] Stakeholder Approval

scdataloader-1.2.1/.github/dependabot.yml

@@ -1,6 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "weekly"

scdataloader-1.2.1/.github/release_message.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-previous_tag=$(git tag --sort=-creatordate | sed -n 2p)
-git shortlog "${previous_tag}.." | sed 's/^./    &/'

scdataloader-1.2.1/.github/rename_project.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-while getopts a:n:u:d: flag
-do
-    case "${flag}" in
-        a) author=${OPTARG};;
-        n) name=${OPTARG};;
-        u) urlname=${OPTARG};;
-        d) description=${OPTARG};;
-    esac
-done
-
-echo "Author: $author";
-echo "Project Name: $name";
-echo "Project URL name: $urlname";
-echo "Description: $description";
-
-echo "Renaming project..."
-
-original_author="jkobject"
-original_name="scdataloader"
-original_urlname="scDataLoader"
-original_description="Awesome scdataloader created by jkobject"
-# for filename in $(find . -name "*.*") 
-for filename in $(git ls-files) 
-do
-    sed -i "s/$original_author/$author/g" $filename
-    sed -i "s/$original_name/$name/g" $filename
-    sed -i "s/$original_urlname/$urlname/g" $filename
-    sed -i "s/$original_description/$description/g" $filename
-    echo "Renamed $filename"
-done
-
-mv scdataloader $name
-
-# This command runs only once on GHA!
-rm -rf .github/template.yml

scdataloader-1.2.1/.github/workflows/main.yml

@@ -1,43 +0,0 @@
-# This is a basic workflow to help you get started with Actions
-
-name: CI
-
-# Controls when the workflow will run
-on:
-  # Triggers the workflow on push or pull request events but only for the main branch
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-  ci:
-    strategy:
-      fail-fast: true
-      matrix:
-        python-version: ["3.10"]
-        os: [ubuntu-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install uv
-        uses: astral-sh/setup-uv@v3
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install project
-        run: make virtualenv
-        continue-on-error: false
-      - name: Run linter
-        run: make lint
-        continue-on-error: false
-      - name: Run tests
-        run: make test
-      - name: "Upload coverage to Codecov"
-        uses: codecov/codecov-action@v4
-        with:
-          fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }}

scdataloader-1.2.1/.github/workflows/release.yml

@@ -1,48 +0,0 @@
-name: Upload Python Package
-
-on:
-  push:
-    # Sequence of patterns matched against refs/tags
-    tags:
-      - "*" # Push events to matching v*, i.e. v1.0, v20.15.10
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-  release:
-    name: Create Release
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          # by default, it uses a depth of 1
-          # this fetches all history so that we can read each commit
-          fetch-depth: 0
-      - name: Generate Changelog
-        run: .github/release_message.sh > release_message.md
-      - name: Release
-        uses: softprops/action-gh-release@v1
-        with:
-          body_path: release_message.md
-
-  deploy:
-    needs: release
-    strategy:
-      fail-fast: true
-      matrix:
-        python-version: ["3.10"]
-        os: [ubuntu-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install uv
-        uses: astral-sh/setup-uv@v3
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Build and publish
-        run: |
-          uv build && uv publish -u __token__ -p ${{ secrets.POETRY_TOKEN }}

scdataloader-1.2.1/ABOUT_THIS_TEMPLATE.md

@@ -1,70 +0,0 @@
-## Structure
-
-Lets take a look at the structure of this template:
-
-```text
-├── Containerfile            # The file to build a container using buildah or docker
-├── CONTRIBUTING.md          # Onboarding instructions for new contributors
-├── docs                     # Documentation site (add more .md files here)
-│   └── index.md             # The index page for the docs site
-├── .github                  # Github metadata for repository
-│   ├── release_message.sh   # A script to generate a release message
-│   └── workflows            # The CI pipeline for Github Actions
-├── .gitignore               # A list of files to ignore when pushing to Github
-├── HISTORY.md               # Auto generated list of changes to the project
-├── LICENSE                  # The license for the project
-├── Makefile                 # A collection of utilities to manage the project
-├── MANIFEST.in              # A list of files to include in a package
-├── mkdocs.yml               # Configuration for documentation site
-├── scdataloader             # The main python package for the project
-│   ├── base.py              # The base module for the project
-│   ├── __init__.py          # This tells Python that this is a package
-│   ├── __main__.py          # The entry point for the project
-│   └── VERSION              # The version for the project is kept in a static file
-├── README.md                # The main readme for the project
-├── setup.py                 # The setup.py file for installing and packaging the project
-├── requirements.txt         # An empty file to hold the requirements for the project
-├── requirements-test.txt    # List of requirements for testing and devlopment
-├── setup.py                 # The setup.py file for installing and packaging the project
-└── tests                    # Unit tests for the project (add mote tests files here)
-    ├── conftest.py          # Configuration, hooks and fixtures for pytest
-    ├── __init__.py          # This tells Python that this is a test package
-    └── test_base.py         # The base test case for the project
-```
-
-### Why to include `tests`, `history` and `Containerfile` as part of the release?
-
-The `MANIFEST.in` file is used to include the files in the release, once the 
-project is released to PyPI all the files listed on MANIFEST.in will be included
-even if the files are static or not related to Python.
-
-Some build systems such as RPM, DEB, AUR for some Linux distributions, and also
-internal repackaging systems tends to run the tests before the packaging is performed.
-
-The Containerfile can be useful to provide a safer execution environment for 
-the project when running on a testing environment.
-
-I added those files to make it easier for packaging in different formats.
-
-## The Makefile
-
-All the utilities for the template and project are on the Makefile
-
-```bash
-❯ make
-Usage: make <target>
-
-Targets:
-help:             ## Show the help.
-install:          ## Install the project in dev mode.
-fmt:              ## Format code using black & isort.
-lint:             ## Run pep8, black, mypy linters.
-test: lint        ## Run tests and generate coverage report.
-watch:            ## Run tests on every change.
-clean:            ## Clean unused files.
-virtualenv:       ## Create a virtual environment.
-release:          ## Create a new tag for release.
-docs:             ## Build the documentation.
-switch-to-poetry: ## Switch to poetry package manager.
-init:             ## Initialize the project based on an application template.
-```

scdataloader-1.2.1/CONTRIBUTING.md

@@ -1,113 +0,0 @@
-# How to develop on this project
-
-scdataloader welcomes contributions from the community.
-
-**You need PYTHON3!**
-
-This instructions are for linux base systems. (Linux, MacOS, BSD, etc.)
-## Setting up your own fork of this repo.
-
-- On github interface click on `Fork` button.
-- Clone your fork of this repo. `git clone git@github.com:YOUR_GIT_USERNAME/scDataLoader.git`
-- Enter the directory `cd scDataLoader`
-- Add upstream repo `git remote add upstream https://github.com/jkobject/scDataLoader`
-
-## Setting up your own virtual environment
-
-Run `make virtualenv` to create a virtual environment.
-then activate it with `source .venv/bin/activate`.
-
-## Install the project in develop mode
-
-Run `make install` to install the project in develop mode.
-
-## Run the tests to ensure everything is working
-
-Run `make test` to run the tests.
-
-## Create a new branch to work on your contribution
-
-Run `git checkout -b my_contribution`
-
-## Make your changes
-
-Edit the files using your preferred editor. (we recommend VIM or VSCode)
-
-## Format the code
-
-Run `make fmt` to format the code.
-
-## Run the linter
-
-Run `make lint` to run the linter.
-
-## Test your changes
-
-Run `make test` to run the tests.
-
-Ensure code coverage report shows `100%` coverage, add tests to your PR.
-
-## Build the docs locally
-
-Run `make docs` to build the docs.
-
-Ensure your new changes are documented.
-
-## Commit your changes
-
-This project uses [conventional git commit messages](https://www.conventionalcommits.org/en/v1.0.0/).
-
-Example: `fix(package): update setup.py arguments 🎉` (emojis are fine too)
-
-## Push your changes to your fork
-
-Run `git push origin my_contribution`
-
-## Submit a pull request
-
-On github interface, click on `Pull Request` button.
-
-Wait CI to run and one of the developers will review your PR.
-## Makefile utilities
-
-This project comes with a `Makefile` that contains a number of useful utility.
-
-```bash 
-❯ make
-Usage: make <target>
-
-Targets:
-help:             ## Show the help.
-install:          ## Install the project in dev mode.
-fmt:              ## Format code using black & isort.
-lint:             ## Run pep8, black, mypy linters.
-test: lint        ## Run tests and generate coverage report.
-watch:            ## Run tests on every change.
-clean:            ## Clean unused files.
-virtualenv:       ## Create a virtual environment.
-release:          ## Create a new tag for release.
-docs:             ## Build the documentation.
-switch-to-poetry: ## Switch to poetry package manager.
-init:             ## Initialize the project based on an application template.
-```
-
-## Making a new release
-
-This project uses [semantic versioning](https://semver.org/) and tags releases with `X.Y.Z`
-Every time a new tag is created and pushed to the remote repo, github actions will
-automatically create a new release on github and trigger a release on PyPI.
-
-For this to work you need to setup a secret called `PIPY_API_TOKEN` on the project settings>secrets, 
-this token can be generated on [pypi.org](https://pypi.org/account/).
-
-To trigger a new release all you need to do is.
-
-1. If you have changes to add to the repo
-    * Make your changes following the steps described above.
-    * Commit your changes following the [conventional git commit messages](https://www.conventionalcommits.org/en/v1.0.0/).
-2. Run the tests to ensure everything is working.
-4. Run `make release` to create a new tag and push it to the remote repo.
-
-the `make release` will ask you the version number to create the tag, ex: type `0.1.1` when you are asked.
-
-> **CAUTION**:  The make release will change local changelog files and commit all the unstaged changes you have.

scdataloader-1.2.1/Containerfile

@@ -1,5 +0,0 @@
-FROM python:3.7-slim
-COPY . /app
-WORKDIR /app
-RUN pip install .
-CMD ["scdataloader"]