brisc 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {brisc-0.2.2 → brisc-0.2.4}/PKG-INFO +1 -1
- {brisc-0.2.2 → brisc-0.2.4}/brisc/harmonize.pyx +34 -30
- {brisc-0.2.2 → brisc-0.2.4}/brisc/norm_factors.pyx +3 -6
- {brisc-0.2.2 → brisc-0.2.4}/brisc/pseudobulk.py +1 -3
- {brisc-0.2.2 → brisc-0.2.4}/brisc/single_cell.py +124 -168
- {brisc-0.2.2 → brisc-0.2.4}/brisc/threadpoolctl.py +4 -1
- {brisc-0.2.2 → brisc-0.2.4}/brisc.egg-info/PKG-INFO +1 -1
- brisc-0.2.4/brisc.egg-info/scm_version.json +8 -0
- brisc-0.2.2/brisc.egg-info/scm_version.json +0 -8
- {brisc-0.2.2 → brisc-0.2.4}/.git_archival.txt +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/.gitattributes +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/.gitignore +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/LICENSE +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/MANIFEST.in +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/README.md +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/__init__.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/colorspacious.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/concatenate.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/cysparse.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/cyutils.pxd +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/cyutils.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/de.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/doublets.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/embed.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/hdf5.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/hvg.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/kmeans.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/knn.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/labels.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/leiden.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/loess.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/normalize.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/pca.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/pseudobulk_and_markers.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/qc.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/snn.pyx +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/sparse.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/type_aliases.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/utils.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc/validated_dict.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc.egg-info/SOURCES.txt +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc.egg-info/dependency_links.txt +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc.egg-info/requires.txt +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc.egg-info/scm_file_list.json +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/brisc.egg-info/top_level.txt +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/pyproject.toml +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/setup.cfg +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/setup.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/conftest.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/make_test_data.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/test_cluster.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/test_de.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/test_embedding.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/test_hvg.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/test_integration.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/test_markers.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/test_neighbors.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/test_normalize.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/test_pca.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/test_pseudobulk.py +0 -0
- {brisc-0.2.2 → brisc-0.2.4}/tests/test_qc.py +0 -0
|
@@ -372,8 +372,7 @@ def harmony(const float[:, ::1] PCs,
|
|
|
372
372
|
cdef float base, kmeans_error, entropy_term, norm, Rij, \
|
|
373
373
|
diversity_penalty, prev_objective, total, Pr_bi, Eij, Oij, Rkj, \
|
|
374
374
|
objective, last_two, old, new, ridge_lambda, factor, beta, \
|
|
375
|
-
two_over_sigma = <float> 2 / sigma, half_sigma = sigma / <float> 2
|
|
376
|
-
exp_neg_two_over_sigma = exp(-two_over_sigma)
|
|
375
|
+
two_over_sigma = <float> 2 / sigma, half_sigma = sigma / <float> 2
|
|
377
376
|
cdef float past_clustering_objectives[3]
|
|
378
377
|
cdef bint compute_objective, compute_Y, converged, cold_start
|
|
379
378
|
cdef str metrics
|
|
@@ -471,8 +470,8 @@ def harmony(const float[:, ::1] PCs,
|
|
|
471
470
|
if tau:
|
|
472
471
|
for i in range(num_batches):
|
|
473
472
|
Pr_b[i] = <float> N_b[i] / num_cells
|
|
474
|
-
base =
|
|
475
|
-
theta_batch[i] = theta * (<float> 1 - base * base)
|
|
473
|
+
base = (<float> N_b[i]) / (num_clusters * tau)
|
|
474
|
+
theta_batch[i] = theta * (<float> 1 - exp(-base * base))
|
|
476
475
|
else:
|
|
477
476
|
for i in range(num_batches):
|
|
478
477
|
Pr_b[i] = <float> N_b[i] / num_cells
|
|
@@ -568,7 +567,9 @@ def harmony(const float[:, ::1] PCs,
|
|
|
568
567
|
kmeans_error = kmeans_error + Rij * (
|
|
569
568
|
<float> 1 - half_sigma *
|
|
570
569
|
distances[thread_index, i, j])
|
|
571
|
-
|
|
570
|
+
# Skip entries of `R` that underflowed to 0
|
|
571
|
+
if Rij > <float> 0:
|
|
572
|
+
entropy_term = entropy_term + Rij * log(Rij)
|
|
572
573
|
distances[thread_index, i, j] = Rij
|
|
573
574
|
beta = piece > slab_piece_offsets[slab]
|
|
574
575
|
matrix_multiply(
|
|
@@ -667,11 +668,12 @@ def harmony(const float[:, ::1] PCs,
|
|
|
667
668
|
# then chunkwise within each block. Note that the full formula
|
|
668
669
|
# for `R` is: `((E + 1) / (O + E + 1)) ** theta *
|
|
669
670
|
# exp(-2 / sigma * (1 - Z @ Y.T))`
|
|
670
|
-
# which we
|
|
671
|
-
# `ratio =
|
|
672
|
-
#
|
|
673
|
-
# `
|
|
674
|
-
#
|
|
671
|
+
# which we calculate as `ratio * exp(distances - 2 / sigma)`
|
|
672
|
+
# where `ratio = ((E + 1) / (O + E + 1)) ** theta` and
|
|
673
|
+
# `distances = 2 / sigma * Z @ Y.T`. Since the rows of `Z` and
|
|
674
|
+
# `Y` have unit norm, the exponent is never positive, so
|
|
675
|
+
# `exp()` can underflow but never overflow, even when `sigma`
|
|
676
|
+
# is small.
|
|
675
677
|
#
|
|
676
678
|
# If we are done clustering or `early_stopping=True`, also
|
|
677
679
|
# compute the first two components of the objective function,
|
|
@@ -701,8 +703,7 @@ def harmony(const float[:, ::1] PCs,
|
|
|
701
703
|
|
|
702
704
|
# Get the chunk of `Z`, the normalized PCs, to
|
|
703
705
|
# process. Calculate the observed-to-expected ratio
|
|
704
|
-
# `ratio =
|
|
705
|
-
# ((E + 1) / (O + E + 1)) ** theta`
|
|
706
|
+
# `ratio = ((E + 1) / (O + E + 1)) ** theta`
|
|
706
707
|
# for the chunk, subtracting the chunk's own
|
|
707
708
|
# contributions to `E` and `O`: `Pr_b` times
|
|
708
709
|
# `R_sums[chunk_index]` and `delta_O[chunk_index]`,
|
|
@@ -727,11 +728,10 @@ def harmony(const float[:, ::1] PCs,
|
|
|
727
728
|
Eij = E[i, j] - \
|
|
728
729
|
Pr_bi * R_sums[chunk_index, j]
|
|
729
730
|
Oij = O[i, j] + delta_O[chunk_index, i, j]
|
|
730
|
-
ratio[chunk_index, i, j] =
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
theta_batch[i])
|
|
731
|
+
ratio[chunk_index, i, j] = pow(
|
|
732
|
+
(Eij + <float> 1) /
|
|
733
|
+
(Oij + Eij + <float> 1),
|
|
734
|
+
theta_batch[i])
|
|
735
735
|
|
|
736
736
|
# Compute `distances = 2 / sigma * Z @ Y.T`
|
|
737
737
|
matrix_multiply(
|
|
@@ -758,7 +758,8 @@ def harmony(const float[:, ::1] PCs,
|
|
|
758
758
|
norm = 0
|
|
759
759
|
for j in range(num_clusters):
|
|
760
760
|
R[k, j] = exp(distances[
|
|
761
|
-
thread_index, i, j]
|
|
761
|
+
thread_index, i, j] -
|
|
762
|
+
two_over_sigma) * \
|
|
762
763
|
ratio[chunk_index, batch_label, j]
|
|
763
764
|
norm += R[k, j]
|
|
764
765
|
norm = <float> 1 / norm
|
|
@@ -779,8 +780,11 @@ def harmony(const float[:, ::1] PCs,
|
|
|
779
780
|
kmeans_error = kmeans_error + Rkj * (
|
|
780
781
|
<float> 1 - half_sigma *
|
|
781
782
|
distances[thread_index, i, j])
|
|
782
|
-
|
|
783
|
-
|
|
783
|
+
# Skip entries of `R` that
|
|
784
|
+
# underflowed to 0
|
|
785
|
+
if Rkj > <float> 0:
|
|
786
|
+
entropy_term = entropy_term + \
|
|
787
|
+
Rkj * log(Rkj)
|
|
784
788
|
kmeans_errors[slot] = kmeans_error
|
|
785
789
|
entropy_terms[slot] = entropy_term
|
|
786
790
|
if compute_Y:
|
|
@@ -889,10 +893,12 @@ def harmony(const float[:, ::1] PCs,
|
|
|
889
893
|
with gil:
|
|
890
894
|
PyErr_CheckSignals()
|
|
891
895
|
|
|
892
|
-
#
|
|
893
|
-
#
|
|
894
|
-
# recompute `R`, `O`, and `E` from
|
|
895
|
-
#
|
|
896
|
+
# Unless this is the last iteration (because we have converged or
|
|
897
|
+
# `max_iterations` has been reached), normalize the corrected
|
|
898
|
+
# embeddings and recompute `R`, `O`, and `E` from them before the
|
|
899
|
+
# next round of clustering (a "cold start"). On the last
|
|
900
|
+
# iteration, leave the embeddings unnormalized, so that the
|
|
901
|
+
# corrected PCs are returned either way.
|
|
896
902
|
converged = prev_objective - objective < \
|
|
897
903
|
tolerance * abs(prev_objective)
|
|
898
904
|
cold_start = not converged and iteration < max_iterations
|
|
@@ -991,10 +997,9 @@ def harmony(const float[:, ::1] PCs,
|
|
|
991
997
|
W[i, k, j] = factor * (
|
|
992
998
|
W[i, k, j] - O[i, k] * intercepts[k, j])
|
|
993
999
|
|
|
994
|
-
# Correct `Z` slab by slab.
|
|
995
|
-
#
|
|
996
|
-
# each slab's
|
|
997
|
-
# `R.T @ Z` for the next `Y`.
|
|
1000
|
+
# Correct `Z` slab by slab. For a cold start, also normalize each
|
|
1001
|
+
# row of `Z` and recompute `R`, each slab's contributions to `O`,
|
|
1002
|
+
# and each slab's partial sum of `R.T @ Z` for the next `Y`.
|
|
998
1003
|
with parallel(num_threads=num_threads_or_chunks):
|
|
999
1004
|
thread_index = threadid()
|
|
1000
1005
|
for slab in prange(num_slabs):
|
|
@@ -1014,7 +1019,7 @@ def harmony(const float[:, ::1] PCs,
|
|
|
1014
1019
|
W[batch_label], Z_chunk[thread_index],
|
|
1015
1020
|
transpose_A=False, transpose_B=False, alpha=-1,
|
|
1016
1021
|
beta=1)
|
|
1017
|
-
if
|
|
1022
|
+
if cold_start:
|
|
1018
1023
|
for i in range(end_row - start_row):
|
|
1019
1024
|
norm = 0
|
|
1020
1025
|
for j in range(num_PCs):
|
|
@@ -1025,7 +1030,6 @@ def harmony(const float[:, ::1] PCs,
|
|
|
1025
1030
|
for j in range(num_PCs):
|
|
1026
1031
|
Z_chunk[thread_index, i, j] = \
|
|
1027
1032
|
Z_chunk[thread_index, i, j] * norm
|
|
1028
|
-
if cold_start:
|
|
1029
1033
|
matrix_multiply(
|
|
1030
1034
|
Z_chunk[thread_index, :end_row - start_row],
|
|
1031
1035
|
Y, distances[thread_index], transpose_A=False,
|
|
@@ -80,8 +80,7 @@ def calc_norm_factors(const numeric[:, ::1] X,
|
|
|
80
80
|
const unsigned ref_sample,
|
|
81
81
|
float[::1] norm_factors,
|
|
82
82
|
sum_type[::1] library_size,
|
|
83
|
-
unsigned num_threads
|
|
84
|
-
const bint is_win):
|
|
83
|
+
unsigned num_threads):
|
|
85
84
|
cdef numeric ref_count, count
|
|
86
85
|
cdef sum_type ref_library_size
|
|
87
86
|
cdef unsigned i, j, n, loL, hiL, loS, hiS, thread_index, start_sample, \
|
|
@@ -126,11 +125,9 @@ def calc_norm_factors(const numeric[:, ::1] X,
|
|
|
126
125
|
log_normalized_X_ref[j] = \
|
|
127
126
|
log2(<float> count * inverse_ref_library_size)
|
|
128
127
|
|
|
129
|
-
# Calculate the normalization factor for each sample
|
|
130
|
-
# path for Windows for floating-point consistency between 1 and multiple
|
|
131
|
-
# threads.
|
|
128
|
+
# Calculate the normalization factor for each sample
|
|
132
129
|
num_threads = min(num_threads, num_samples)
|
|
133
|
-
if num_threads <= 1
|
|
130
|
+
if num_threads <= 1:
|
|
134
131
|
logR_buffer.resize(num_genes)
|
|
135
132
|
absE_buffer.resize(num_genes)
|
|
136
133
|
logR_rank_buffer.resize(num_genes)
|
|
@@ -4,7 +4,6 @@ import os
|
|
|
4
4
|
import polars as pl
|
|
5
5
|
import re
|
|
6
6
|
import shutil
|
|
7
|
-
import sys
|
|
8
7
|
from collections.abc import Iterable
|
|
9
8
|
from itertools import islice, pairwise
|
|
10
9
|
from pathlib import Path
|
|
@@ -4252,11 +4251,10 @@ class Pseudobulk:
|
|
|
4252
4251
|
|
|
4253
4252
|
# Calculate norm factors
|
|
4254
4253
|
norm_factors = np.empty(num_samples, dtype=np.float32)
|
|
4255
|
-
is_win = sys.platform == 'win32'
|
|
4256
4254
|
calc_norm_factors(X=X, logratio_trim=logratio_trim, sum_trim=sum_trim,
|
|
4257
4255
|
A_cutoff=A_cutoff, ref_sample=ref_sample,
|
|
4258
4256
|
norm_factors=norm_factors, library_size=library_size,
|
|
4259
|
-
num_threads=num_threads
|
|
4257
|
+
num_threads=num_threads)
|
|
4260
4258
|
|
|
4261
4259
|
return norm_factors # this is actually `library_size * norm_factors`
|
|
4262
4260
|
|
|
@@ -11391,103 +11391,111 @@ class SingleCell:
|
|
|
11391
11391
|
thread_sums = np.empty((num_threads, max(
|
|
11392
11392
|
dataset._X.shape[1] for dataset in datasets), 3),
|
|
11393
11393
|
dtype=np.uint64)
|
|
11394
|
-
|
|
11395
|
-
|
|
11396
|
-
|
|
11397
|
-
|
|
11398
|
-
|
|
11399
|
-
|
|
11400
|
-
|
|
11401
|
-
|
|
11402
|
-
|
|
11403
|
-
|
|
11404
|
-
|
|
11405
|
-
|
|
11406
|
-
|
|
11407
|
-
|
|
11408
|
-
|
|
11409
|
-
|
|
11410
|
-
|
|
11411
|
-
|
|
11412
|
-
|
|
11413
|
-
|
|
11414
|
-
|
|
11415
|
-
|
|
11416
|
-
|
|
11417
|
-
|
|
11418
|
-
|
|
11419
|
-
|
|
11420
|
-
|
|
11421
|
-
|
|
11422
|
-
num_cells = X.shape[0]
|
|
11394
|
+
with threadpool_limits(
|
|
11395
|
+
1, user_api=None if num_threads == 1 else 'blas'):
|
|
11396
|
+
for X, cell_mask, gene_indices in batches:
|
|
11397
|
+
num_dataset_genes = X.shape[1]
|
|
11398
|
+
mean = np.empty(num_dataset_genes, dtype=np.float32)
|
|
11399
|
+
var = np.empty(num_dataset_genes, dtype=np.float32)
|
|
11400
|
+
nonzero_count = np.empty(num_dataset_genes, dtype=np.uint32)
|
|
11401
|
+
is_csr = isinstance(X, csr_array)
|
|
11402
|
+
if is_csr:
|
|
11403
|
+
# For CSR, `cell_mask` is either `None` (all cells) or a
|
|
11404
|
+
# pre-computed int64 array of cell indices
|
|
11405
|
+
if cell_mask is None:
|
|
11406
|
+
cell_indices = np.array([], dtype=np.int64)
|
|
11407
|
+
num_cells = X.shape[0]
|
|
11408
|
+
else:
|
|
11409
|
+
cell_indices = cell_mask
|
|
11410
|
+
num_cells = len(cell_indices)
|
|
11411
|
+
counts_sum = np.empty(num_dataset_genes, dtype=np.uint64)
|
|
11412
|
+
squared_counts_sum = np.empty(num_dataset_genes,
|
|
11413
|
+
dtype=np.uint64)
|
|
11414
|
+
gene_mean_and_variance_csr(
|
|
11415
|
+
data=X.data, indices=X.indices, indptr=X.indptr,
|
|
11416
|
+
cell_indices=cell_indices, num_cells=num_cells,
|
|
11417
|
+
num_dataset_genes=num_dataset_genes,
|
|
11418
|
+
thread_sums=thread_sums, counts_sum=counts_sum,
|
|
11419
|
+
squared_counts_sum=squared_counts_sum, mean=mean,
|
|
11420
|
+
var=var, nonzero_count=nonzero_count,
|
|
11421
|
+
num_threads=num_threads)
|
|
11423
11422
|
else:
|
|
11424
|
-
|
|
11425
|
-
|
|
11426
|
-
|
|
11427
|
-
|
|
11428
|
-
|
|
11429
|
-
|
|
11430
|
-
|
|
11431
|
-
|
|
11432
|
-
|
|
11433
|
-
|
|
11434
|
-
|
|
11435
|
-
|
|
11423
|
+
if cell_mask is None:
|
|
11424
|
+
cell_mask = np.array([], dtype=bool)
|
|
11425
|
+
num_cells = X.shape[0]
|
|
11426
|
+
else:
|
|
11427
|
+
# The cast to int is necessary for floating-point
|
|
11428
|
+
# identical results: in NumPy ≥ 2.0, a Python int is a
|
|
11429
|
+
# "weak" scalar (int * float32 array -> float32) but a
|
|
11430
|
+
# NumPy integer is "strong" (np.int64 * float32 array
|
|
11431
|
+
# -> float64 since
|
|
11432
|
+
# `np.promote_types(int64, float32) == float64`).
|
|
11433
|
+
num_cells = int(cell_mask.sum())
|
|
11434
|
+
gene_mean_and_variance_csc(
|
|
11435
|
+
data=X.data, indices=X.indices, indptr=X.indptr,
|
|
11436
|
+
cell_mask=cell_mask, num_cells=num_cells,
|
|
11437
|
+
num_dataset_genes=num_dataset_genes, mean=mean,
|
|
11438
|
+
var=var, nonzero_count=nonzero_count,
|
|
11439
|
+
num_threads=num_threads)
|
|
11436
11440
|
|
|
11437
|
-
|
|
11438
|
-
|
|
11439
|
-
|
|
11440
|
-
|
|
11441
|
-
|
|
11442
|
-
|
|
11443
|
-
|
|
11444
|
-
|
|
11445
|
-
|
|
11446
|
-
|
|
11447
|
-
|
|
11448
|
-
|
|
11449
|
-
|
|
11450
|
-
|
|
11451
|
-
|
|
11452
|
-
|
|
11453
|
-
|
|
11454
|
-
|
|
11455
|
-
|
|
11456
|
-
|
|
11457
|
-
|
|
11458
|
-
|
|
11459
|
-
|
|
11460
|
-
|
|
11461
|
-
|
|
11462
|
-
|
|
11463
|
-
|
|
11464
|
-
|
|
11465
|
-
|
|
11466
|
-
|
|
11467
|
-
|
|
11468
|
-
|
|
11469
|
-
|
|
11470
|
-
|
|
11471
|
-
|
|
11472
|
-
|
|
11473
|
-
|
|
11474
|
-
|
|
11475
|
-
|
|
11476
|
-
|
|
11477
|
-
|
|
11478
|
-
|
|
11479
|
-
|
|
11480
|
-
|
|
11481
|
-
|
|
11482
|
-
|
|
11483
|
-
|
|
11484
|
-
|
|
11485
|
-
|
|
11486
|
-
|
|
11487
|
-
norm_gene_var
|
|
11488
|
-
|
|
11489
|
-
|
|
11490
|
-
|
|
11441
|
+
not_constant = var > 0
|
|
11442
|
+
y = np.log10(var[not_constant])
|
|
11443
|
+
x = np.log10(mean[not_constant])
|
|
11444
|
+
model = loess(x, y, span=span)
|
|
11445
|
+
try:
|
|
11446
|
+
model.fit()
|
|
11447
|
+
except ValueError as e:
|
|
11448
|
+
error_message = (
|
|
11449
|
+
f'LOESS model fitting failed; this typically happens '
|
|
11450
|
+
f'when there are very few cells (under ~1000), or '
|
|
11451
|
+
f'when batch_column is specified and certain batches '
|
|
11452
|
+
f'have very few cells')
|
|
11453
|
+
if batch_column is not None:
|
|
11454
|
+
error_message += \
|
|
11455
|
+
': consider not specifying batch_column'
|
|
11456
|
+
raise ValueError(error_message) from e
|
|
11457
|
+
|
|
11458
|
+
estimated_variance = np.zeros(num_dataset_genes,
|
|
11459
|
+
dtype=np.float32)
|
|
11460
|
+
estimated_variance[not_constant] = model.outputs.fitted_values
|
|
11461
|
+
estimated_stddev = np.sqrt(10 ** estimated_variance)
|
|
11462
|
+
clip_val = mean + estimated_stddev * np.sqrt(num_cells,
|
|
11463
|
+
dtype=np.float32)
|
|
11464
|
+
batch_counts_sum = np.empty(num_dataset_genes,
|
|
11465
|
+
dtype=np.float32)
|
|
11466
|
+
squared_batch_counts_sum = np.empty(num_dataset_genes,
|
|
11467
|
+
dtype=np.float32)
|
|
11468
|
+
if is_csr:
|
|
11469
|
+
clipped_sum_csr(
|
|
11470
|
+
data=X.data, indices=X.indices, indptr=X.indptr,
|
|
11471
|
+
num_cells=num_cells,
|
|
11472
|
+
num_dataset_genes=num_dataset_genes,
|
|
11473
|
+
cell_indices=cell_indices, clip_val=clip_val,
|
|
11474
|
+
thread_sums=thread_sums, counts_sum=counts_sum,
|
|
11475
|
+
squared_counts_sum=squared_counts_sum,
|
|
11476
|
+
batch_counts_sum=batch_counts_sum,
|
|
11477
|
+
squared_batch_counts_sum=squared_batch_counts_sum,
|
|
11478
|
+
num_threads=num_threads)
|
|
11479
|
+
else:
|
|
11480
|
+
clipped_sum_csc(
|
|
11481
|
+
data=X.data, indices=X.indices, indptr=X.indptr,
|
|
11482
|
+
cell_mask=cell_mask, clip_val=clip_val,
|
|
11483
|
+
batch_counts_sum=batch_counts_sum,
|
|
11484
|
+
squared_batch_counts_sum=squared_batch_counts_sum,
|
|
11485
|
+
num_threads=num_threads)
|
|
11486
|
+
norm_gene_var = pl.Series(
|
|
11487
|
+
(1 / ((num_cells - 1) * np.square(estimated_stddev))) *
|
|
11488
|
+
((num_cells * np.square(mean)) + squared_batch_counts_sum -
|
|
11489
|
+
2 * batch_counts_sum * mean))
|
|
11490
|
+
|
|
11491
|
+
# `norm_gene_var` and `nonzero_count` are with respect to the
|
|
11492
|
+
# genes in `dataset.var_names`; if needed, map them to the
|
|
11493
|
+
# genes in `genes_in_all_datasets`
|
|
11494
|
+
if gene_indices is not None:
|
|
11495
|
+
norm_gene_var = norm_gene_var[gene_indices]
|
|
11496
|
+
nonzero_count = nonzero_count[gene_indices.to_numpy()]
|
|
11497
|
+
norm_gene_vars.append(norm_gene_var)
|
|
11498
|
+
total_nonzero_count += nonzero_count
|
|
11491
11499
|
|
|
11492
11500
|
rank = pl.exclude('gene').rank('min', descending=True)
|
|
11493
11501
|
final_rank = pl.struct(
|
|
@@ -13325,7 +13333,7 @@ class SingleCell:
|
|
|
13325
13333
|
oversampling_factor: int | np.integer | float |
|
|
13326
13334
|
np.floating = 1,
|
|
13327
13335
|
chunk_size_kmeans: int | np.integer | None = None,
|
|
13328
|
-
max_clustering_iterations: int | np.integer =
|
|
13336
|
+
max_clustering_iterations: int | np.integer = 4,
|
|
13329
13337
|
block_proportion: int | float | np.integer |
|
|
13330
13338
|
np.floating = 0.05,
|
|
13331
13339
|
tolerance: int | float | np.integer | np.floating = 0.01,
|
|
@@ -13347,52 +13355,12 @@ class SingleCell:
|
|
|
13347
13355
|
multiple batches of the same dataset, with
|
|
13348
13356
|
[Harmony2](https://github.com/immunogenomics/harmony).
|
|
13349
13357
|
|
|
13350
|
-
|
|
13351
|
-
|
|
13352
|
-
|
|
13353
|
-
|
|
13354
|
-
|
|
13355
|
-
|
|
13356
|
-
total cells. It iterates over each block, subtracting the contribution
|
|
13357
|
-
of the cells in the block to the observed and expected cluster-batch
|
|
13358
|
-
co-occurence matrices `O` and `E`, re-calculating the soft-clustering
|
|
13359
|
-
assignments `R` based on the residual `O` and `E`, then adding back the
|
|
13360
|
-
contribution of the cells in the block to `O` and `E` based on the new
|
|
13361
|
-
`R`. This approach resists straightforward parallelization because
|
|
13362
|
-
updates to `R` for future blocks depend on updates to `O` and `E` from
|
|
13363
|
-
previous blocks, leading to convergence failure if naively
|
|
13364
|
-
parallelized. Instead, we parallelize within blocks by dividing them
|
|
13365
|
-
into chunks of `chunk_size_Harmony` cells. We process each chunk in
|
|
13366
|
-
parallel, subtracting only the `O` and `E` contributions of the chunk
|
|
13367
|
-
itself, updating `R` for the chunk, and, after processing all chunks in
|
|
13368
|
-
the block, add back the `O` and `E` contributions for all chunks based
|
|
13369
|
-
on the updated `R`. Updating `O` and `E` at the end of each block
|
|
13370
|
-
(rather than after processing every cell in the dataset) ensures
|
|
13371
|
-
convergence, while the inner chunking enables parallelization without
|
|
13372
|
-
disrupting convergence. To use the original implementation's strategy
|
|
13373
|
-
of updating `R`, `O`, and `E` for each block as a whole, set
|
|
13374
|
-
`chunk_size_Harmony` to at least the block size (`block_proportion`
|
|
13375
|
-
times the total number of cells, rounded up to a multiple of 16), at
|
|
13376
|
-
the cost of no longer parallelizing the update of `R`.
|
|
13377
|
-
|
|
13378
|
-
Second, we reduce the default number of clustering iterations
|
|
13379
|
-
(Harmony's inner loop) from 20 to 5, but always complete all 5
|
|
13380
|
-
iterations without early stopping based on convergence. In practice,
|
|
13381
|
-
the largest changes to the Harmony objective function result from
|
|
13382
|
-
updating the PCs (outer loop), not updating the soft-clustering
|
|
13383
|
-
assignments (inner loop). Our implementation makes updating the PCs
|
|
13384
|
-
sufficiently fast that there are rapidly diminishing returns from
|
|
13385
|
-
performing lots of clustering updates, versus just skipping directly to
|
|
13386
|
-
updating the PCs after a few clustering iterations. To use the original
|
|
13387
|
-
implementation's default of 20 iterations with early stopping, specify
|
|
13388
|
-
`early_stopping=True, max_clustering_iterations=20`.
|
|
13389
|
-
|
|
13390
|
-
Third, we use random k-means initialization, like version 1 of Harmony,
|
|
13391
|
-
rather than Harmony2's kmeans++ initialization. We implement a parallel
|
|
13392
|
-
version of kmeans++ called [k-means||](https://arxiv.org/abs/1203.6402),
|
|
13393
|
-
but find it does not improve the accuracy of label transfer relative to
|
|
13394
|
-
random initialization and increases runtime. kmeans|| initialization can
|
|
13395
|
-
be enabled with `kmeans_barbar=True`.
|
|
13358
|
+
Unlike Harmony2, our implementation parallelizes the clustering step
|
|
13359
|
+
within each block of cells, by dividing each block into chunks of
|
|
13360
|
+
`chunk_size_Harmony` cells that are updated in parallel. It also uses
|
|
13361
|
+
random k-means initialization rather than k-means++ (initialization
|
|
13362
|
+
with k-means||, a parallel version of k-means++, can be enabled with
|
|
13363
|
+
`kmeans_barbar=True`).
|
|
13396
13364
|
|
|
13397
13365
|
Args:
|
|
13398
13366
|
others: the other SingleCell datasets to harmonize this one with.
|
|
@@ -13440,20 +13408,15 @@ class SingleCell:
|
|
|
13440
13408
|
30 and 3000 cells.
|
|
13441
13409
|
max_iterations: the maximum number of iterations to run Harmony
|
|
13442
13410
|
for, if convergence is not achieved. Defaults to
|
|
13443
|
-
10
|
|
13444
|
-
|
|
13445
|
-
use as many iterations as necessary to achieve
|
|
13446
|
-
convergence.
|
|
13411
|
+
10. Set to `None` to use as many iterations as
|
|
13412
|
+
necessary to achieve convergence.
|
|
13447
13413
|
num_kmeans_iterations: the maximum number of iterations of
|
|
13448
|
-
k-means clustering to perform
|
|
13449
|
-
|
|
13450
|
-
|
|
13414
|
+
k-means clustering to perform to
|
|
13415
|
+
initialize the cluster centroids, stopping
|
|
13416
|
+
early if a relative convergence of
|
|
13451
13417
|
`kmeans_tolerance` is reached. Defaults to
|
|
13452
|
-
25,
|
|
13453
|
-
|
|
13454
|
-
unlike these packages, only one
|
|
13455
|
-
initialization is tried rather than 10 to
|
|
13456
|
-
reduce runtime.
|
|
13418
|
+
25, whereas Harmony2 always runs 10
|
|
13419
|
+
iterations without early stopping.
|
|
13457
13420
|
kmeans_tolerance: the relative change in inertia (the sum of
|
|
13458
13421
|
squared distances from each cell to its assigned
|
|
13459
13422
|
centroid) used to determine whether to stop
|
|
@@ -13496,19 +13459,13 @@ class SingleCell:
|
|
|
13496
13459
|
clustering step within each Harmony
|
|
13497
13460
|
iteration for, or the maximum number of
|
|
13498
13461
|
iterations if `early_stopping=True`.
|
|
13499
|
-
|
|
13500
|
-
convergence of the clustering is not
|
|
13462
|
+
Convergence of the clustering is not
|
|
13501
13463
|
checked unless `early_stopping=True`.
|
|
13502
|
-
Defaults to
|
|
13503
|
-
|
|
13504
|
-
|
|
13505
|
-
|
|
13506
|
-
|
|
13507
|
-
Must be 4 or more when
|
|
13508
|
-
`early_stopping=True`, since Harmony's
|
|
13509
|
-
clustering convergence check requires
|
|
13510
|
-
knowing the errors from the past 3
|
|
13511
|
-
iterations.
|
|
13464
|
+
Defaults to 4 iterations. Must be 4 or
|
|
13465
|
+
more when `early_stopping=True`, since
|
|
13466
|
+
the clustering convergence check
|
|
13467
|
+
requires knowing the errors from the
|
|
13468
|
+
past 3 iterations.
|
|
13512
13469
|
block_proportion: the proportion of cells to use in each batch
|
|
13513
13470
|
update in the clustering step; must be greater
|
|
13514
13471
|
than zero and less than or equal to 1
|
|
@@ -13517,8 +13474,7 @@ class SingleCell:
|
|
|
13517
13474
|
`max_iterations` iterations
|
|
13518
13475
|
early_stopping: whether to stop clustering before
|
|
13519
13476
|
`max_clustering_iterations` iterations if
|
|
13520
|
-
convergence is reached
|
|
13521
|
-
Harmony algorithm
|
|
13477
|
+
convergence is reached.
|
|
13522
13478
|
clustering_tolerance: the relative tolerance used to determine
|
|
13523
13479
|
whether to stop clustering before
|
|
13524
13480
|
`max_clustering_iterations` iterations. Only
|
|
@@ -160,7 +160,10 @@ class _FlexiBLAS(_Library):
|
|
|
160
160
|
|
|
161
161
|
class _MKL(_Library):
|
|
162
162
|
name = 'mkl'
|
|
163
|
-
|
|
163
|
+
# Also match MKL's interface layers (e.g. `libmkl_intel_lp64()`), which
|
|
164
|
+
# export the threading functions when MKL is linked without libmkl_rt
|
|
165
|
+
filename_prefixes = \
|
|
166
|
+
'libmkl_rt', 'mkl_rt', 'libmkl_intel_', 'libmkl_gf_', 'libblas'
|
|
164
167
|
check_symbols = 'MKL_Set_Num_Threads',
|
|
165
168
|
|
|
166
169
|
def get(self):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|