discotoolkit 1.3.1__tar.gz → 1.3.2__tar.gz

{discotoolkit-1.3.1 → discotoolkit-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: discotoolkit
-Version: 1.3.1
+Version: 1.3.2
 Summary: DISCOtoolkit is an python package that allows users to access data and use the tools provided by the DISCO database.
 Home-page: https://disco.bii.a-star.edu.sg/
 Author: Li Mengwei, Rom Uddamvathanak
@@ -11,12 +11,11 @@ Project-URL: Issues, https://github.com/JinmiaoChenLab/DISCOtoolkit_py/issues
 Project-URL: DISCO, https://disco.bii.a-star.edu.sg/v1/
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: pandas<3,>=1.4.2
+Requires-Dist: pandas>=1.4.2
 Requires-Dist: numpy>=1.21.6
 Requires-Dist: scanpy>=1.9.3
 Requires-Dist: scipy>=1.10.1
 Requires-Dist: joblib>=1.1.0
-Requires-Dist: pandarallel>=1.6.5
 Requires-Dist: requests>=2.28.1
 Requires-Dist: colorcet>=3.0.1
 Requires-Dist: leidenalg
@@ -43,7 +42,7 @@ Dynamic: summary
 -->
 [![Documentation](https://img.shields.io/badge/docs-disco.bii.a--star.edu.sg-blue)](https://disco.bii.a-star.edu.sg/v1/docs/toolkit/guide/) [![Downloads](https://static.pepy.tech/personalized-badge/discotoolkit?period=total&units=international_system&left_color=black&right_color=orange&left_text=Downloads)](https://pepy.tech/project/discotoolkit) [![PyPI version](https://img.shields.io/pypi/v/discotoolkit)](https://pypi.org/project/discotoolkit)
 
-# DISCOtoolkit 1.3.1
+# DISCOtoolkit 1.3.2
 
 DISCOtoolkit is a Python package for accessing the data and tools of the [DISCO database](https://disco.bii.a-star.edu.sg/) (DISCO v1). Read the documentation, with tutorials and the API reference, at [disco.bii.a-star.edu.sg/v1/docs/toolkit/guide](https://disco.bii.a-star.edu.sg/v1/docs/toolkit/guide/).
 
@@ -84,7 +83,7 @@ pip install -e .
 
 `pip install discotoolkit` and a local install give the same code when the versions match. The difference is the source: PyPI delivers a released, packaged copy, while a local install uses the files in your folder.
 
-Dependencies (installed automatically): numpy, pandas, scanpy, scipy, joblib, pandarallel, requests, colorcet, leidenalg, h5py, matplotlib, seaborn.
+Dependencies (installed automatically): numpy, pandas, scanpy, scipy, joblib, requests, colorcet, leidenalg, h5py, matplotlib, seaborn.
 
 We recommend a virtual environment, for example with miniconda:
 

{discotoolkit-1.3.1 → discotoolkit-1.3.2}/README.md

@@ -8,7 +8,7 @@
 -->
 [![Documentation](https://img.shields.io/badge/docs-disco.bii.a--star.edu.sg-blue)](https://disco.bii.a-star.edu.sg/v1/docs/toolkit/guide/) [![Downloads](https://static.pepy.tech/personalized-badge/discotoolkit?period=total&units=international_system&left_color=black&right_color=orange&left_text=Downloads)](https://pepy.tech/project/discotoolkit) [![PyPI version](https://img.shields.io/pypi/v/discotoolkit)](https://pypi.org/project/discotoolkit)
 
-# DISCOtoolkit 1.3.1
+# DISCOtoolkit 1.3.2
 
 DISCOtoolkit is a Python package for accessing the data and tools of the [DISCO database](https://disco.bii.a-star.edu.sg/) (DISCO v1). Read the documentation, with tutorials and the API reference, at [disco.bii.a-star.edu.sg/v1/docs/toolkit/guide](https://disco.bii.a-star.edu.sg/v1/docs/toolkit/guide/).
 
@@ -49,7 +49,7 @@ pip install -e .
 
 `pip install discotoolkit` and a local install give the same code when the versions match. The difference is the source: PyPI delivers a released, packaged copy, while a local install uses the files in your folder.
 
-Dependencies (installed automatically): numpy, pandas, scanpy, scipy, joblib, pandarallel, requests, colorcet, leidenalg, h5py, matplotlib, seaborn.
+Dependencies (installed automatically): numpy, pandas, scanpy, scipy, joblib, requests, colorcet, leidenalg, h5py, matplotlib, seaborn.
 
 We recommend a virtual environment, for example with miniconda:
 

{discotoolkit-1.3.1 → discotoolkit-1.3.2}/discotoolkit/CELLiD.py

@@ -13,10 +13,43 @@ from scipy import stats
 # from fast_fisher import fast_fisher_exact, odds_ratio
 from scipy.stats.contingency import odds_ratio
 from joblib import Parallel, delayed  # multiprocessing library
-from pandarallel import pandarallel  # multiprocessing library
 
 pd.options.mode.chained_assignment = None
 
+
+def _spearman_matrix(ref_data: pd.DataFrame, rna) -> np.ndarray:
+    """Spearman correlation of every reference column with every user column (reference x user).
+
+    Spearman is Pearson on ranks, so with no missing values the whole table is one matrix product and needs no
+    process pool. Columns with a missing value fall back to pandas' own pairwise-complete Series.corr, which
+    gives the same numbers as before for them. A constant column has no defined correlation and gives NaN.
+    """
+    ref = pd.DataFrame(ref_data)
+    user = pd.DataFrame(rna)
+    ref_values = ref.to_numpy(dtype=float)
+    user_values = user.to_numpy(dtype=float)
+    result = np.full((ref_values.shape[1], user_values.shape[1]), np.nan)
+
+    ref_ok = ~np.isnan(ref_values).any(axis=0)
+    user_ok = ~np.isnan(user_values).any(axis=0)
+
+    def standardised_ranks(frame: pd.DataFrame, columns: np.ndarray) -> np.ndarray:
+        ranks = frame.iloc[:, np.flatnonzero(columns)].rank(axis=0, method="average").to_numpy(dtype=float)
+        ranks = ranks - ranks.mean(axis=0)
+        with np.errstate(divide="ignore", invalid="ignore"):
+            return ranks / np.sqrt((ranks**2).sum(axis=0))
+
+    if ref_ok.any() and user_ok.any():
+        block = standardised_ranks(ref, ref_ok).T @ standardised_ranks(user, user_ok)
+        result[np.ix_(np.flatnonzero(ref_ok), np.flatnonzero(user_ok))] = np.clip(block, -1.0, 1.0)
+
+    # Anything touching a missing value is done pair by pair, as pandas always did.
+    for j in range(user_values.shape[1]):
+        for i in range(ref_values.shape[1]):
+            if not (ref_ok[i] and user_ok[j]):
+                result[i, j] = pd.Series(ref_values[:, i]).corr(pd.Series(user_values[:, j]), method="spearman")
+    return result
+
 # import variable and class from other script
 from .GlobalVariable import logging, api_url, timeout
 from .DiscoClass import FilterData, Filter
@@ -183,9 +216,6 @@ def CELLiD_cluster(
         Pandas DataFrame: return the Pandas DataFrame along with the correlation score.
     """
 
-    # initialize the pandarallel for parallel processing
-    pandarallel.initialize(nb_workers=ncores)
-
     # check for the input data
     if not (isinstance(rna, pd.DataFrame) or isinstance(rna, np.ndarray)):
         logging.error("the rna must be a pandas DataFrame or Numpy Array")
@@ -247,30 +277,8 @@ def CELLiD_cluster(
     rna = rna.loc[list(genes)]
     ref_data = ref_data.loc[list(genes)]
 
-    # nested apply function to compute correlation for each cell type between the user and reference database
-    def each_ref_correlation(input, ref_data):
-        input = list(input)
-        res = ref_data.parallel_apply(basic_correlation, input=input, axis=0)
-        return res
-
-    # apply the correlation to all the columns in user data
-    def basic_correlation(ref, input):
-        # predicted = stats.spearmanr(
-        #     np.asarray(ref), np.asarray(input), alternative = "two-sided", nan_policy='omit'
-        # )[0]
-        predicted = pd.Series(list(ref)).corr(pd.Series(list(input)), method="spearman")
-
-        return predicted
-
-    # get the predicted correlation as in pandas DataFrame
-    # predicted_cell = pd.DataFrame(
-    #     generate_correlation_map(ref_data.to_numpy().T, rna.to_numpy().T)
-    # )
-    predicted_cell = rna.apply(
-        each_ref_correlation, result_type="expand", axis=0, ref_data=ref_data
-    )
-    predicted_cell = pd.DataFrame(predicted_cell)
-
+    # correlation of every user column with every reference cell type (rows: cell types, columns: user clusters)
+    predicted_cell = pd.DataFrame(_spearman_matrix(ref_data, rna), columns=rna.columns)
     # set the index to be the celltype based on the reference_data
     predicted_cell.index = ref_data.columns
 

{discotoolkit-1.3.1 → discotoolkit-1.3.2}/discotoolkit/__init__.py

@@ -5,4 +5,4 @@ from .CELLiD import *
 from .GeneSearch import *
 from .GlobalVariable import set_server, get_server
 
-__version__ = "1.3.1"
+__version__ = "1.3.2"

{discotoolkit-1.3.1 → discotoolkit-1.3.2}/discotoolkit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: discotoolkit
-Version: 1.3.1
+Version: 1.3.2
 Summary: DISCOtoolkit is an python package that allows users to access data and use the tools provided by the DISCO database.
 Home-page: https://disco.bii.a-star.edu.sg/
 Author: Li Mengwei, Rom Uddamvathanak
@@ -11,12 +11,11 @@ Project-URL: Issues, https://github.com/JinmiaoChenLab/DISCOtoolkit_py/issues
 Project-URL: DISCO, https://disco.bii.a-star.edu.sg/v1/
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: pandas<3,>=1.4.2
+Requires-Dist: pandas>=1.4.2
 Requires-Dist: numpy>=1.21.6
 Requires-Dist: scanpy>=1.9.3
 Requires-Dist: scipy>=1.10.1
 Requires-Dist: joblib>=1.1.0
-Requires-Dist: pandarallel>=1.6.5
 Requires-Dist: requests>=2.28.1
 Requires-Dist: colorcet>=3.0.1
 Requires-Dist: leidenalg
@@ -43,7 +42,7 @@ Dynamic: summary
 -->
 [![Documentation](https://img.shields.io/badge/docs-disco.bii.a--star.edu.sg-blue)](https://disco.bii.a-star.edu.sg/v1/docs/toolkit/guide/) [![Downloads](https://static.pepy.tech/personalized-badge/discotoolkit?period=total&units=international_system&left_color=black&right_color=orange&left_text=Downloads)](https://pepy.tech/project/discotoolkit) [![PyPI version](https://img.shields.io/pypi/v/discotoolkit)](https://pypi.org/project/discotoolkit)
 
-# DISCOtoolkit 1.3.1
+# DISCOtoolkit 1.3.2
 
 DISCOtoolkit is a Python package for accessing the data and tools of the [DISCO database](https://disco.bii.a-star.edu.sg/) (DISCO v1). Read the documentation, with tutorials and the API reference, at [disco.bii.a-star.edu.sg/v1/docs/toolkit/guide](https://disco.bii.a-star.edu.sg/v1/docs/toolkit/guide/).
 
@@ -84,7 +83,7 @@ pip install -e .
 
 `pip install discotoolkit` and a local install give the same code when the versions match. The difference is the source: PyPI delivers a released, packaged copy, while a local install uses the files in your folder.
 
-Dependencies (installed automatically): numpy, pandas, scanpy, scipy, joblib, pandarallel, requests, colorcet, leidenalg, h5py, matplotlib, seaborn.
+Dependencies (installed automatically): numpy, pandas, scanpy, scipy, joblib, requests, colorcet, leidenalg, h5py, matplotlib, seaborn.
 
 We recommend a virtual environment, for example with miniconda:
 

{discotoolkit-1.3.1 → discotoolkit-1.3.2}/discotoolkit.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ discotoolkit.egg-info/SOURCES.txt
 discotoolkit.egg-info/dependency_links.txt
 discotoolkit.egg-info/requires.txt
 discotoolkit.egg-info/top_level.txt
+tests/test_cellid_correlation.py
 tests/test_docs.py
 tests/test_gene_search.py
 tests/test_metadata_helpers.py

{discotoolkit-1.3.1 → discotoolkit-1.3.2}/discotoolkit.egg-info/requires.txt

@@ -1,9 +1,8 @@
-pandas<3,>=1.4.2
+pandas>=1.4.2
 numpy>=1.21.6
 scanpy>=1.9.3
 scipy>=1.10.1
 joblib>=1.1.0
-pandarallel>=1.6.5
 requests>=2.28.1
 colorcet>=3.0.1
 leidenalg

{discotoolkit-1.3.1 → discotoolkit-1.3.2}/requirements.txt

@@ -1,9 +1,8 @@
-pandas>=1.4.2,<3
+pandas>=1.4.2
 numpy>=1.21.6
 scanpy>=1.9.3
 scipy>=1.10.1
 joblib>=1.1.0
-pandarallel>=1.6.5
 requests>=2.28.1
 colorcet>=3.0.1
 leidenalg

{discotoolkit-1.3.1 → discotoolkit-1.3.2}/setup.py

@@ -6,7 +6,7 @@ with open("requirements.txt", "r") as f:
 
 setup(
     name='discotoolkit',
-    version="1.3.1",
+    version="1.3.2",
     url='https://disco.bii.a-star.edu.sg/',
     author='Li Mengwei, Rom Uddamvathanak',
     author_email='uddamvathanak_rom@immunol.a-star.edu.sg',

discotoolkit-1.3.2/tests/test_cellid_correlation.py

@@ -0,0 +1,91 @@
+"""
+CELLiD's correlation step: the matrix calculation that replaced pandarallel must give the numbers pandas'
+own pairwise Spearman gives (ties, missing values, constant columns included), and CELLiD_cluster must
+annotate a small made-up reference correctly. No network.
+
+    python tests/test_cellid_correlation.py
+"""
+import os
+import sys
+
+import numpy as np
+import pandas as pd
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+from discotoolkit.CELLiD import CELLiD_cluster, _spearman_matrix  # noqa: E402
+
+
+def pandas_pairwise(ref: pd.DataFrame, user: pd.DataFrame) -> np.ndarray:
+    """What the code did before: one Series.corr(method="spearman") per pair."""
+    return np.array([[pd.Series(ref.iloc[:, i].to_numpy()).corr(pd.Series(user.iloc[:, j].to_numpy()), method="spearman")
+                      for j in range(user.shape[1])] for i in range(ref.shape[1])])
+
+
+def same(a, b):
+    assert a.shape == b.shape, (a.shape, b.shape)
+    assert np.array_equal(np.isnan(a), np.isnan(b)), "NaN positions differ"
+    assert np.allclose(a[~np.isnan(a)], b[~np.isnan(b)], atol=1e-12), np.nanmax(np.abs(a - b))
+
+
+def test_matches_pandas_on_continuous_data():
+    rng = np.random.default_rng(1)
+    ref = pd.DataFrame(rng.normal(size=(300, 12)))
+    user = pd.DataFrame(rng.normal(size=(300, 5)))
+    same(_spearman_matrix(ref, user), pandas_pairwise(ref, user))
+
+
+def test_matches_pandas_with_many_ties_like_expression_data():
+    rng = np.random.default_rng(2)
+    ref = pd.DataFrame(rng.poisson(0.4, size=(400, 10)).astype(float))
+    user = pd.DataFrame(rng.poisson(0.4, size=(400, 4)).astype(float))
+    same(_spearman_matrix(ref, user), pandas_pairwise(ref, user))
+
+
+def test_missing_values_use_pairwise_complete_observations():
+    rng = np.random.default_rng(3)
+    ref = pd.DataFrame(rng.normal(size=(200, 6)))
+    user = pd.DataFrame(rng.normal(size=(200, 3)))
+    ref.iloc[5, 2] = np.nan
+    user.iloc[17, 1] = np.nan
+    same(_spearman_matrix(ref, user), pandas_pairwise(ref, user))
+
+
+def test_a_constant_column_has_no_correlation():
+    rng = np.random.default_rng(4)
+    ref = pd.DataFrame(rng.normal(size=(50, 3)))
+    ref[1] = 7.0
+    user = pd.DataFrame(rng.normal(size=(50, 2)))
+    result = _spearman_matrix(ref, user)
+    assert np.isnan(result[1]).all() and not np.isnan(result[[0, 2]]).any()
+
+
+def test_cellid_cluster_annotates_a_small_reference():
+    rng = np.random.default_rng(5)
+    genes = ["G%d" % i for i in range(300)]
+    types = ["B cell--blood", "T cell--blood", "Macrophage--lung", "Fibroblast--lung"]
+    ref = pd.DataFrame(rng.poisson(1.0, size=(300, 4)).astype(float), index=genes, columns=types)
+    # every cluster is a noisy copy of one reference cell type
+    picks = [2, 0, 3]
+    user = pd.DataFrame({str(c): ref.iloc[:, t] + rng.normal(0, 0.2, size=300) for c, t in enumerate(picks)}, index=genes)
+    deg = pd.DataFrame([(t, g) for t in types for g in genes[:150]], columns=["group", "gene"])
+    out = CELLiD_cluster(user, ref_data=ref, ref_deg=deg, n_predict=1, ncores=1)
+    assert list(out["predicted_cell_type_1"]) == ["Macrophage", "B cell", "Fibroblast"], out
+    assert list(out["source_atlas_1"]) == ["lung", "blood", "lung"], out
+
+
+def _run_all():
+    tests = [(n, f) for n, f in sorted(globals().items()) if n.startswith("test_") and callable(f)]
+    failed = 0
+    for name, fn in tests:
+        try:
+            fn()
+            print("  PASS  %s" % name)
+        except Exception as error:
+            failed += 1
+            print("  FAIL  %s\n          %s: %s" % (name, type(error).__name__, error))
+    print("\n%d/%d passed" % (len(tests) - failed, len(tests)))
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(_run_all())