truecell 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- truecell/__init__.py +218 -0
- truecell/_clara.py +469 -0
- truecell/_sparse.py +67 -0
- truecell/_types.py +15 -0
- truecell/_utils.py +72 -0
- truecell/aggregate.py +156 -0
- truecell/anchors.py +934 -0
- truecell/assay.py +444 -0
- truecell/assay5.py +691 -0
- truecell/clustering.py +261 -0
- truecell/command.py +124 -0
- truecell/compat/__init__.py +3 -0
- truecell/compat/anndata.py +247 -0
- truecell/composition.py +100 -0
- truecell/datasets.py +622 -0
- truecell/dimreduc.py +192 -0
- truecell/generics.py +397 -0
- truecell/glmpca.py +506 -0
- truecell/graph.py +180 -0
- truecell/hto.py +361 -0
- truecell/integration.py +283 -0
- truecell/io.py +134 -0
- truecell/jackstraw.py +259 -0
- truecell/lazy.py +381 -0
- truecell/logmap.py +96 -0
- truecell/mapping.py +289 -0
- truecell/markers.py +819 -0
- truecell/mixins/__init__.py +3 -0
- truecell/mixins/key_mixin.py +30 -0
- truecell/mixscape.py +762 -0
- truecell/module_score.py +278 -0
- truecell/multimodal.py +333 -0
- truecell/multiseq.py +272 -0
- truecell/neighbor.py +129 -0
- truecell/neighbors.py +175 -0
- truecell/plotting.py +1832 -0
- truecell/preprocessing.py +959 -0
- truecell/py.typed +0 -0
- truecell/reduction.py +478 -0
- truecell/sctransform.py +679 -0
- truecell/sketch.py +563 -0
- truecell/spatial/__init__.py +49 -0
- truecell/spatial/analysis.py +239 -0
- truecell/spatial/base.py +95 -0
- truecell/spatial/centroids.py +146 -0
- truecell/spatial/fov.py +274 -0
- truecell/spatial/loaders.py +384 -0
- truecell/spatial/molecules.py +95 -0
- truecell/spatial/segmentation.py +140 -0
- truecell/spatial/variable_features.py +515 -0
- truecell/spatial/visium.py +288 -0
- truecell/transfer.py +394 -0
- truecell/truecell.py +682 -0
- truecell/umap.py +149 -0
- truecell-0.9.0.dist-info/METADATA +537 -0
- truecell-0.9.0.dist-info/RECORD +58 -0
- truecell-0.9.0.dist-info/WHEEL +4 -0
- truecell-0.9.0.dist-info/licenses/LICENSE +21 -0
truecell/__init__.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""truecell — Python port of satijalab/seurat-object (v5.4.0)."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version as _metadata_version
|
|
4
|
+
|
|
5
|
+
from .assay import Assay, create_assay_object
|
|
6
|
+
from .assay5 import Assay5, StdAssay, create_assay5_object
|
|
7
|
+
from .command import TruecellCommand, log_truecell_command
|
|
8
|
+
from .dimreduc import DimReduc
|
|
9
|
+
from .graph import Graph, as_graph
|
|
10
|
+
from .jackstraw import JackStrawData, jack_straw, score_jackstraw
|
|
11
|
+
from .logmap import LogMap
|
|
12
|
+
from .mixins import KeyMixin
|
|
13
|
+
from .neighbor import Neighbor
|
|
14
|
+
from .truecell import Truecell, create_truecell_object
|
|
15
|
+
from .preprocessing import (
|
|
16
|
+
normalize_data,
|
|
17
|
+
find_variable_features,
|
|
18
|
+
scale_data,
|
|
19
|
+
percentage_feature_set,
|
|
20
|
+
)
|
|
21
|
+
from .reduction import run_pca, run_ica, run_spca, run_tsne
|
|
22
|
+
from .glmpca import glm_pca
|
|
23
|
+
from .neighbors import find_neighbors
|
|
24
|
+
from .multimodal import find_multi_modal_neighbors
|
|
25
|
+
from .clustering import find_clusters
|
|
26
|
+
from .umap import run_umap
|
|
27
|
+
from .integration import run_harmony, integrate_layers
|
|
28
|
+
from .anchors import (
|
|
29
|
+
find_integration_anchors,
|
|
30
|
+
integrate_data,
|
|
31
|
+
integrate_embeddings,
|
|
32
|
+
IntegrationAnchors,
|
|
33
|
+
)
|
|
34
|
+
from .transfer import (
|
|
35
|
+
find_transfer_anchors,
|
|
36
|
+
transfer_data,
|
|
37
|
+
TransferAnchors,
|
|
38
|
+
)
|
|
39
|
+
from .mapping import map_query, project_umap
|
|
40
|
+
from .sketch import sketch_data, project_data, leverage_score
|
|
41
|
+
from .lazy import LazyMatrix, write_lazy_matrix, open_lazy_matrix, is_lazy
|
|
42
|
+
from .hto import hto_demux
|
|
43
|
+
from .multiseq import multiseq_demux
|
|
44
|
+
from .mixscape import calc_perturb_sig, run_mixscape, mixscape_lda
|
|
45
|
+
from .markers import find_markers, find_all_markers, find_conserved_markers
|
|
46
|
+
from .aggregate import aggregate_expression
|
|
47
|
+
from .sctransform import sctransform
|
|
48
|
+
from .module_score import add_module_score, cell_cycle_scoring, CC_GENES
|
|
49
|
+
from .spatial import (
|
|
50
|
+
Centroids,
|
|
51
|
+
FOV,
|
|
52
|
+
Molecules,
|
|
53
|
+
ScaleFactors,
|
|
54
|
+
Segmentation,
|
|
55
|
+
SpatialImage,
|
|
56
|
+
VisiumV2,
|
|
57
|
+
create_centroids,
|
|
58
|
+
create_fov,
|
|
59
|
+
create_fovs,
|
|
60
|
+
create_molecules,
|
|
61
|
+
create_segmentation,
|
|
62
|
+
build_niche_assay,
|
|
63
|
+
find_spatially_variable_features,
|
|
64
|
+
get_tissue_coordinates,
|
|
65
|
+
local_neighborhood,
|
|
66
|
+
nearest_neighbor_distance,
|
|
67
|
+
spatial_knn,
|
|
68
|
+
load_cosmx,
|
|
69
|
+
load_merscope,
|
|
70
|
+
load_visium,
|
|
71
|
+
load_xenium,
|
|
72
|
+
)
|
|
73
|
+
from .composition import composition_test
|
|
74
|
+
from . import generics
|
|
75
|
+
from . import plotting
|
|
76
|
+
from .plotting import (
|
|
77
|
+
vln_plot,
|
|
78
|
+
feature_plot,
|
|
79
|
+
dim_plot,
|
|
80
|
+
elbow_plot,
|
|
81
|
+
feature_scatter,
|
|
82
|
+
variable_feature_plot,
|
|
83
|
+
viz_dim_loadings,
|
|
84
|
+
dim_heatmap,
|
|
85
|
+
do_heatmap,
|
|
86
|
+
ridge_plot,
|
|
87
|
+
dot_plot,
|
|
88
|
+
image_dim_plot,
|
|
89
|
+
image_feature_plot,
|
|
90
|
+
spatial_dim_plot,
|
|
91
|
+
spatial_feature_plot,
|
|
92
|
+
plot_perturb_score,
|
|
93
|
+
mixscape_heatmap,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
try:
|
|
97
|
+
__version__ = _metadata_version("truecell")
|
|
98
|
+
except PackageNotFoundError: # pragma: no cover - needs an uninstalled source tree
|
|
99
|
+
# A source tree on sys.path with no installed distribution: the package still
|
|
100
|
+
# imports and works, only its version is unknowable. PEP 440-valid on purpose,
|
|
101
|
+
# so `packaging.version.Version(truecell.__version__)` parses on this path too.
|
|
102
|
+
__version__ = "0.0.0+unknown"
|
|
103
|
+
|
|
104
|
+
__all__ = [
|
|
105
|
+
# Core classes
|
|
106
|
+
"Truecell",
|
|
107
|
+
"Assay",
|
|
108
|
+
"Assay5",
|
|
109
|
+
"StdAssay",
|
|
110
|
+
"DimReduc",
|
|
111
|
+
"Graph",
|
|
112
|
+
"Neighbor",
|
|
113
|
+
"JackStrawData",
|
|
114
|
+
"LogMap",
|
|
115
|
+
"KeyMixin",
|
|
116
|
+
"TruecellCommand",
|
|
117
|
+
# Spatial
|
|
118
|
+
"SpatialImage",
|
|
119
|
+
"Centroids",
|
|
120
|
+
"Segmentation",
|
|
121
|
+
"Molecules",
|
|
122
|
+
"FOV",
|
|
123
|
+
"VisiumV2",
|
|
124
|
+
"ScaleFactors",
|
|
125
|
+
# Factories
|
|
126
|
+
"create_truecell_object",
|
|
127
|
+
"create_assay_object",
|
|
128
|
+
"create_assay5_object",
|
|
129
|
+
"create_centroids",
|
|
130
|
+
"create_segmentation",
|
|
131
|
+
"create_molecules",
|
|
132
|
+
"create_fov",
|
|
133
|
+
"create_fovs",
|
|
134
|
+
# Spatial analysis
|
|
135
|
+
"get_tissue_coordinates",
|
|
136
|
+
"spatial_knn",
|
|
137
|
+
"nearest_neighbor_distance",
|
|
138
|
+
"local_neighborhood",
|
|
139
|
+
"build_niche_assay",
|
|
140
|
+
"find_spatially_variable_features",
|
|
141
|
+
"composition_test",
|
|
142
|
+
# Spatial loaders
|
|
143
|
+
"load_xenium",
|
|
144
|
+
"load_visium",
|
|
145
|
+
"load_cosmx",
|
|
146
|
+
"load_merscope",
|
|
147
|
+
"as_graph",
|
|
148
|
+
"log_truecell_command",
|
|
149
|
+
# Analysis pipeline (mirrors Seurat's top-level functions)
|
|
150
|
+
"normalize_data",
|
|
151
|
+
"find_variable_features",
|
|
152
|
+
"scale_data",
|
|
153
|
+
"percentage_feature_set",
|
|
154
|
+
"run_pca",
|
|
155
|
+
"run_ica",
|
|
156
|
+
"run_spca",
|
|
157
|
+
"run_tsne",
|
|
158
|
+
"glm_pca",
|
|
159
|
+
"find_neighbors",
|
|
160
|
+
"find_multi_modal_neighbors",
|
|
161
|
+
"find_clusters",
|
|
162
|
+
"run_umap",
|
|
163
|
+
"run_harmony",
|
|
164
|
+
"integrate_layers",
|
|
165
|
+
"find_integration_anchors",
|
|
166
|
+
"integrate_data",
|
|
167
|
+
"integrate_embeddings",
|
|
168
|
+
"IntegrationAnchors",
|
|
169
|
+
"find_transfer_anchors",
|
|
170
|
+
"transfer_data",
|
|
171
|
+
"TransferAnchors",
|
|
172
|
+
"map_query",
|
|
173
|
+
"project_umap",
|
|
174
|
+
"sketch_data",
|
|
175
|
+
"project_data",
|
|
176
|
+
"leverage_score",
|
|
177
|
+
"LazyMatrix",
|
|
178
|
+
"write_lazy_matrix",
|
|
179
|
+
"open_lazy_matrix",
|
|
180
|
+
"is_lazy",
|
|
181
|
+
"hto_demux",
|
|
182
|
+
"multiseq_demux",
|
|
183
|
+
"calc_perturb_sig",
|
|
184
|
+
"run_mixscape",
|
|
185
|
+
"mixscape_lda",
|
|
186
|
+
"find_markers",
|
|
187
|
+
"find_all_markers",
|
|
188
|
+
"find_conserved_markers",
|
|
189
|
+
"aggregate_expression",
|
|
190
|
+
"jack_straw",
|
|
191
|
+
"score_jackstraw",
|
|
192
|
+
"sctransform",
|
|
193
|
+
"add_module_score",
|
|
194
|
+
"cell_cycle_scoring",
|
|
195
|
+
"CC_GENES",
|
|
196
|
+
# Generic functions module
|
|
197
|
+
"generics",
|
|
198
|
+
# Plotting module
|
|
199
|
+
"plotting",
|
|
200
|
+
"vln_plot",
|
|
201
|
+
"feature_plot",
|
|
202
|
+
"dim_plot",
|
|
203
|
+
"elbow_plot",
|
|
204
|
+
"feature_scatter",
|
|
205
|
+
"variable_feature_plot",
|
|
206
|
+
"viz_dim_loadings",
|
|
207
|
+
"dim_heatmap",
|
|
208
|
+
"do_heatmap",
|
|
209
|
+
"ridge_plot",
|
|
210
|
+
"dot_plot",
|
|
211
|
+
"image_dim_plot",
|
|
212
|
+
"image_feature_plot",
|
|
213
|
+
"spatial_dim_plot",
|
|
214
|
+
"spatial_feature_plot",
|
|
215
|
+
"plot_perturb_score",
|
|
216
|
+
"mixscape_heatmap",
|
|
217
|
+
"__version__",
|
|
218
|
+
]
|
truecell/_clara.py
ADDED
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
"""CLARA — Clustering LARge Applications (Kaufman & Rousseeuw), ported from R.
|
|
2
|
+
|
|
3
|
+
A faithful port of the ``clara`` C routine in R's **cluster** package (2.1.8.2,
|
|
4
|
+
``src/clara.c``), which is what Seurat's ``HTODemux`` reaches for by default.
|
|
5
|
+
CLARA is k-medoids for datasets too big to run PAM on directly: rather than
|
|
6
|
+
building the full n x n dissimilarity matrix, it repeatedly draws a small
|
|
7
|
+
sub-sample, runs PAM on that, assigns *every* object to the resulting medoids,
|
|
8
|
+
and keeps whichever sub-sample gave the lowest total dissimilarity.
|
|
9
|
+
|
|
10
|
+
Why a port and not a library
|
|
11
|
+
----------------------------
|
|
12
|
+
The output is fed to ``HTODemux``'s background fit, so it has to agree with what
|
|
13
|
+
R produces, and the details that decide agreement are all non-obvious:
|
|
14
|
+
|
|
15
|
+
* **The RNG is clara's own**, not R's. ``clara(rngR = FALSE)`` — the default, and
|
|
16
|
+
what Seurat uses — draws from a 16-bit LCG seeded to 0, so the result is
|
|
17
|
+
deterministic given the data alone and ``set.seed`` has no effect on it. R's
|
|
18
|
+
``HTODemux`` calls ``set.seed(seed)`` regardless; for the clara path that call
|
|
19
|
+
does nothing.
|
|
20
|
+
* **The swap rule is not PAM's.** At ``pamLike = FALSE`` (again the default)
|
|
21
|
+
``bswap2`` uses the pre-2011 clara update, which the C source itself flags as
|
|
22
|
+
"seems a bit illogical". A textbook PAM, or a k-medoids library, silently
|
|
23
|
+
disagrees here.
|
|
24
|
+
* **Ties break inconsistently, and on purpose.** BUILD takes the *last* candidate
|
|
25
|
+
attaining the maximum (the C carries a comment that ``<`` instead of ``<=``
|
|
26
|
+
does *not* work); SWAP and the final assignment take the *first*. Getting these
|
|
27
|
+
backwards perturbs cluster membership on data with duplicate points — exactly
|
|
28
|
+
what hashtag counts are full of.
|
|
29
|
+
* **Cluster numbering is by first appearance**, not by medoid index: ``selec``
|
|
30
|
+
permutes the medoids into the order their clusters are first encountered while
|
|
31
|
+
scanning objects in order.
|
|
32
|
+
|
|
33
|
+
Rounding is part of the algorithm
|
|
34
|
+
---------------------------------
|
|
35
|
+
clara takes a swap on *any* improvement below zero, and R really does accept
|
|
36
|
+
swaps worth ``-2.2e-16``. A one-ulp difference in a single distance can therefore
|
|
37
|
+
flip a swap, and with it the winning sub-sample and the entire clustering. Sums
|
|
38
|
+
here are accumulated in the same sequence as the C loops (see ``_pairwise_dys``,
|
|
39
|
+
``_bswap2`` and ``_selec``) rather than handed to ``np.sum``, whose pairwise
|
|
40
|
+
reordering is enough to change the result.
|
|
41
|
+
|
|
42
|
+
That sensitivity has a consequence worth stating plainly: **R's clara is not
|
|
43
|
+
reproducible across CPU architectures.** ``clara.c`` built for arm64 contracts
|
|
44
|
+
``clk += d*d`` into a fused multiply-add — one rounding — while the same source
|
|
45
|
+
built for baseline x86_64, whose ISA has no FMA, rounds twice. The two binaries
|
|
46
|
+
return materially different clusterings for a few percent of inputs, so there is
|
|
47
|
+
no single "R answer" to match. This port follows plain IEEE double arithmetic,
|
|
48
|
+
which is what numpy gives everywhere and what ``clara.c`` gives on x86_64;
|
|
49
|
+
against that reference it is exact. Where the two R builds agree — the overwhelming
|
|
50
|
+
majority of inputs, and every realistic hashtag panel tested — truecell agrees too.
|
|
51
|
+
Only the Euclidean metric is ported, the only one ``HTODemux`` asks for.
|
|
52
|
+
|
|
53
|
+
``tests/test_hto.py`` pins this module's output to clustering vectors captured
|
|
54
|
+
from real ``cluster::clara`` runs rather than to a re-derivation of the algorithm
|
|
55
|
+
in Python. Those fixtures use values on a 1/4 grid so their arithmetic is exact
|
|
56
|
+
and they cannot drift with any of the above.
|
|
57
|
+
"""
|
|
58
|
+
from __future__ import annotations
|
|
59
|
+
|
|
60
|
+
import bisect
|
|
61
|
+
|
|
62
|
+
import numpy as np
|
|
63
|
+
|
|
64
|
+
__all__ = ["clara", "clara_sampsize"]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# ----------------------------------------------------------------------
|
|
68
|
+
# Public API
|
|
69
|
+
# ----------------------------------------------------------------------
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def clara_sampsize(n: int, k: int) -> int:
|
|
73
|
+
"""R's default ``sampsize`` for :func:`clara`: ``min(n, 40 + 2 * k)``."""
|
|
74
|
+
return int(min(n, 40 + 2 * k))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def clara(
|
|
78
|
+
x: np.ndarray,
|
|
79
|
+
k: int,
|
|
80
|
+
samples: int = 5,
|
|
81
|
+
sampsize: int | None = None,
|
|
82
|
+
) -> np.ndarray:
|
|
83
|
+
"""Cluster ``x`` into ``k`` medoid-based groups, as R's ``cluster::clara``.
|
|
84
|
+
|
|
85
|
+
Mirrors ``clara(x, k, samples = samples, sampsize = sampsize)`` at the R
|
|
86
|
+
defaults Seurat relies on — ``metric = "euclidean"``, ``rngR = FALSE``,
|
|
87
|
+
``pamLike = FALSE``, ``stand = FALSE`` — and reproduces its cluster
|
|
88
|
+
assignments exactly.
|
|
89
|
+
|
|
90
|
+
There is no ``seed`` argument because clara at ``rngR = FALSE`` does not take
|
|
91
|
+
one: its sampling is driven by a built-in generator that always starts from
|
|
92
|
+
the same state, so the result is a deterministic function of ``x``, ``k``,
|
|
93
|
+
``samples`` and ``sampsize``.
|
|
94
|
+
|
|
95
|
+
Parameters
|
|
96
|
+
----------
|
|
97
|
+
x : ``(n_observations, n_features)`` float array. Note this is
|
|
98
|
+
observations-by-features, the orientation R's ``clara`` expects —
|
|
99
|
+
callers holding a features-by-cells matrix must transpose first.
|
|
100
|
+
k : number of clusters; ``1 <= k <= n - 1``.
|
|
101
|
+
samples : number of sub-samples to draw (R's default is 5; Seurat passes
|
|
102
|
+
100). Ignored when ``sampsize >= n``, where one sample is the whole
|
|
103
|
+
dataset and further draws could not differ.
|
|
104
|
+
sampsize : observations per sub-sample; defaults to ``min(n, 40 + 2 * k)``.
|
|
105
|
+
|
|
106
|
+
Returns
|
|
107
|
+
-------
|
|
108
|
+
numpy.ndarray
|
|
109
|
+
``(n_observations,)`` array of **0-based** cluster labels. R's ``clara``
|
|
110
|
+
returns 1-based labels; the shift is the only intentional difference from
|
|
111
|
+
it, for consistency with the rest of truecell (and scikit-learn).
|
|
112
|
+
"""
|
|
113
|
+
x = np.asarray(x, dtype=float)
|
|
114
|
+
if x.ndim != 2:
|
|
115
|
+
raise ValueError(f"clara() needs a 2-D array; got shape {x.shape}.")
|
|
116
|
+
n, jpp = x.shape
|
|
117
|
+
|
|
118
|
+
k = int(k)
|
|
119
|
+
if k < 1 or k > n - 1:
|
|
120
|
+
raise ValueError(
|
|
121
|
+
f"The number of clusters should be at least 1 and at most n-1; "
|
|
122
|
+
f"got k={k} with n={n}."
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
nsam = clara_sampsize(n, k) if sampsize is None else int(sampsize)
|
|
126
|
+
if nsam < max(2, k + 1):
|
|
127
|
+
raise ValueError(
|
|
128
|
+
f"'sampsize' should be at least {max(2, k + 1)} "
|
|
129
|
+
f"= max(2, 1 + number of clusters); got {nsam}."
|
|
130
|
+
)
|
|
131
|
+
if nsam > n:
|
|
132
|
+
raise ValueError(
|
|
133
|
+
f"'sampsize' = {nsam} should not be larger than the number of "
|
|
134
|
+
f"objects, {n}."
|
|
135
|
+
)
|
|
136
|
+
samples = int(samples)
|
|
137
|
+
if samples < 1:
|
|
138
|
+
raise ValueError(f"'samples' should be at least 1; got {samples}.")
|
|
139
|
+
|
|
140
|
+
return _cl_clara(x, k, samples, nsam)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
# ----------------------------------------------------------------------
|
|
144
|
+
# The built-in RNG (clara.c :: randm)
|
|
145
|
+
# ----------------------------------------------------------------------
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class _Randm:
|
|
149
|
+
"""clara's own generator — a 16-bit LCG, always started from 0.
|
|
150
|
+
|
|
151
|
+
Verbatim from ``clara.c``::
|
|
152
|
+
|
|
153
|
+
*nrun = (*nrun * 5761 + 999) & 0177777;
|
|
154
|
+
return ((double) (*nrun) / 65536.);
|
|
155
|
+
|
|
156
|
+
``0177777`` is octal for 65535, so the mask is ``% 65536``. The period is
|
|
157
|
+
65536, which the C source acknowledges is short and deems "good enough". This
|
|
158
|
+
is *not* R's RNG: ``set.seed`` cannot reach it.
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
__slots__ = ("nrun",)
|
|
162
|
+
|
|
163
|
+
def __init__(self) -> None:
|
|
164
|
+
self.nrun = 0
|
|
165
|
+
|
|
166
|
+
def __call__(self) -> float:
|
|
167
|
+
self.nrun = (self.nrun * 5761 + 999) & 0o177777
|
|
168
|
+
return self.nrun / 65536.0
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# ----------------------------------------------------------------------
|
|
172
|
+
# Distances
|
|
173
|
+
# ----------------------------------------------------------------------
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _pairwise_dys(x: np.ndarray, nsel: np.ndarray) -> np.ndarray:
|
|
177
|
+
"""Full ``(nsam, nsam)`` Euclidean distance matrix for the selected rows.
|
|
178
|
+
|
|
179
|
+
Stands in for ``dysta2`` plus C's ``ind_2`` condensed indexing. ``ind_2(i, i)``
|
|
180
|
+
returns 0 and ``dys[0]`` is held at 0. permanently, so a dense matrix with a
|
|
181
|
+
zero diagonal is exactly equivalent and far easier to read.
|
|
182
|
+
|
|
183
|
+
The accumulation loops over features rather than calling a vectorized norm:
|
|
184
|
+
C sums ``clk += (x[lj] - x[kj])^2`` one feature at a time, and numpy's
|
|
185
|
+
pairwise summation would otherwise add them in a different order. The values
|
|
186
|
+
agree to well under a rounding error either way, but ``bswap2`` compares
|
|
187
|
+
distances with ``==``, so matching the exact float is worth the loop — it runs
|
|
188
|
+
once per feature, not per pair.
|
|
189
|
+
"""
|
|
190
|
+
sub = x[nsel]
|
|
191
|
+
nsam = sub.shape[0]
|
|
192
|
+
acc = np.zeros((nsam, nsam), dtype=float)
|
|
193
|
+
for j in range(sub.shape[1]):
|
|
194
|
+
diff = sub[:, j][:, None] - sub[:, j][None, :]
|
|
195
|
+
acc += diff * diff
|
|
196
|
+
# dysta2 scales by jpp/npres, which is exactly 1.0 without missing data.
|
|
197
|
+
return np.sqrt(acc)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _dist_to_medoids(x: np.ndarray, medoids: np.ndarray) -> np.ndarray:
|
|
201
|
+
"""``(n, k)`` *squared* Euclidean distances from every row to each medoid.
|
|
202
|
+
|
|
203
|
+
Squared, because both ``selec`` and ``resul`` compare sums-of-squares and only
|
|
204
|
+
take the square root once a winner is chosen. Feature-wise accumulation for
|
|
205
|
+
the same reason as :func:`_pairwise_dys`.
|
|
206
|
+
"""
|
|
207
|
+
med = x[medoids]
|
|
208
|
+
acc = np.zeros((x.shape[0], med.shape[0]), dtype=float)
|
|
209
|
+
for j in range(x.shape[1]):
|
|
210
|
+
diff = x[:, j][:, None] - med[:, j][None, :]
|
|
211
|
+
acc += diff * diff
|
|
212
|
+
return acc
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
# ----------------------------------------------------------------------
|
|
216
|
+
# PAM on a sub-sample (clara.c :: bswap2)
|
|
217
|
+
# ----------------------------------------------------------------------
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _bswap2(kk: int, dys: np.ndarray, s: float) -> np.ndarray:
|
|
221
|
+
"""PAM's BUILD then SWAP over one sub-sample; returns a medoid mask.
|
|
222
|
+
|
|
223
|
+
``dys`` is the ``(nsam, nsam)`` distance matrix from :func:`_pairwise_dys` and
|
|
224
|
+
``s`` its maximum. Ported from ``bswap2`` at ``pam_like = FALSE``, i.e. the
|
|
225
|
+
swap clara has used since before 2011, which is *not* the one ``pam()`` uses.
|
|
226
|
+
"""
|
|
227
|
+
n = dys.shape[0]
|
|
228
|
+
s = s * 1.1 + 1.0 # strictly larger than every dissimilarity
|
|
229
|
+
|
|
230
|
+
# ---- BUILD: greedily seed kk medoids ----
|
|
231
|
+
nrepr = np.zeros(n, dtype=bool)
|
|
232
|
+
dysma = np.full(n, s)
|
|
233
|
+
|
|
234
|
+
for _ in range(kk):
|
|
235
|
+
# beter[i] = sum_j max(0, dysma[j] - dys[i, j]), the gain from adding i.
|
|
236
|
+
beter = np.zeros(n)
|
|
237
|
+
for j in range(n):
|
|
238
|
+
cmd = dysma[j] - dys[:, j]
|
|
239
|
+
beter += np.where(cmd > 0.0, cmd, 0.0)
|
|
240
|
+
|
|
241
|
+
# C scans i ascending under `if (ammax <= beter[i])`, so among candidates
|
|
242
|
+
# tied at the maximum the LAST one wins. The source explicitly notes that
|
|
243
|
+
# tightening `<=` to `<` breaks the algorithm.
|
|
244
|
+
cand = np.flatnonzero(~nrepr)
|
|
245
|
+
nmax = cand[np.flatnonzero(beter[cand] == beter[cand].max())[-1]]
|
|
246
|
+
|
|
247
|
+
nrepr[nmax] = True
|
|
248
|
+
dysma = np.minimum(dysma, dys[nmax])
|
|
249
|
+
|
|
250
|
+
if kk == 1:
|
|
251
|
+
return nrepr
|
|
252
|
+
|
|
253
|
+
# ---- SWAP: exchange a medoid for a non-medoid while that helps ----
|
|
254
|
+
while True:
|
|
255
|
+
med = np.flatnonzero(nrepr)
|
|
256
|
+
# dysma[j] = d(j, closest medoid); dysmb[j] = d(j, 2nd closest).
|
|
257
|
+
# Sorting reproduces C's running two-smallest scan: with kk >= 2 both
|
|
258
|
+
# entries are always overwritten (s exceeds every distance), and ties
|
|
259
|
+
# yield identical values whatever the order.
|
|
260
|
+
near = np.sort(dys[med], axis=0)
|
|
261
|
+
dysma, dysmb = near[0], near[1]
|
|
262
|
+
|
|
263
|
+
non_med = np.flatnonzero(~nrepr)
|
|
264
|
+
|
|
265
|
+
# dz[h, i] = the change in total dissimilarity from swapping medoid i out
|
|
266
|
+
# for non-medoid h. Laid out h-major because that is the order C scans in,
|
|
267
|
+
# which decides who wins a tie; see the argmin below.
|
|
268
|
+
#
|
|
269
|
+
# Accumulated one j at a time rather than summed along an axis, because C
|
|
270
|
+
# adds the terms in that order and the residue is load-bearing: the terms
|
|
271
|
+
# routinely cancel to a mathematical zero, and whether the leftover lands
|
|
272
|
+
# a hair below zero or exactly on it decides whether the swap happens at
|
|
273
|
+
# all. R really does accept swaps worth -2.2e-16.
|
|
274
|
+
dz = np.zeros((len(non_med), len(med)))
|
|
275
|
+
for j in range(n):
|
|
276
|
+
dj_i = dys[med, j][None, :] # (1, n_med) -- d(i, j)
|
|
277
|
+
dj_h = dys[non_med, j][:, None] # (n_non_med, 1) -- d(h, j)
|
|
278
|
+
|
|
279
|
+
# The pam_like = FALSE branch. Note `dysmb[j] > dys[i, j]` tests the
|
|
280
|
+
# *removed* medoid's distance where pam() tests the candidate's --
|
|
281
|
+
# and inside this branch dys[i, j] == dysma[j], so it is really asking
|
|
282
|
+
# whether the 2nd-closest is further than the closest, which it all
|
|
283
|
+
# but always is. The C comment calls this "a bit illogical"; it is
|
|
284
|
+
# also what R computes, so it is what we compute.
|
|
285
|
+
on_i = dj_i == dysma[j]
|
|
286
|
+
small = np.where(dysmb[j] > dj_i, dj_h, dysmb[j])
|
|
287
|
+
dz += np.where(
|
|
288
|
+
on_i,
|
|
289
|
+
-dysma[j] + small,
|
|
290
|
+
np.where(dj_h < dysma[j], -dysma[j] + dj_h, 0.0),
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
# C nests `for h { for i { if (dzsky > dz) ... } }` from dzsky = 1, so the
|
|
294
|
+
# winner is the first pair attaining the minimum scanning h-major -- which
|
|
295
|
+
# is exactly argmin on the row-major flat view. Note this is the opposite
|
|
296
|
+
# tie-break to BUILD's, and h-major rather than i-major: both matter only
|
|
297
|
+
# on ties, and hashtag counts tie often.
|
|
298
|
+
flat = int(np.argmin(dz))
|
|
299
|
+
if dz.flat[flat] >= 0.0: # no improving swap left
|
|
300
|
+
return nrepr
|
|
301
|
+
h_pos, i_pos = divmod(flat, len(med))
|
|
302
|
+
nrepr[non_med[h_pos]] = True
|
|
303
|
+
nrepr[med[i_pos]] = False
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
# ----------------------------------------------------------------------
|
|
307
|
+
# Whole-dataset assignment (clara.c :: selec / resul)
|
|
308
|
+
# ----------------------------------------------------------------------
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _assign(x: np.ndarray, medoids: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
|
312
|
+
"""Assign every row of ``x`` to its nearest medoid.
|
|
313
|
+
|
|
314
|
+
Returns ``(labels, dist)`` with ``labels`` indexing into ``medoids`` and
|
|
315
|
+
``dist`` the Euclidean distance to the chosen one. Shared by ``selec`` (which
|
|
316
|
+
sums ``dist`` into the sample's score) and ``resul`` (which keeps the labels),
|
|
317
|
+
since the two do the same work in the no-missing-data case.
|
|
318
|
+
"""
|
|
319
|
+
dsq = _dist_to_medoids(x, medoids)
|
|
320
|
+
# Both C loops replace the incumbent only on a strictly smaller distance, so
|
|
321
|
+
# the first medoid attaining the minimum wins -- which is np.argmin's rule.
|
|
322
|
+
labels = np.argmin(dsq, axis=1)
|
|
323
|
+
# A medoid is assigned to its own cluster even where some other medoid sits
|
|
324
|
+
# at distance 0 from it (duplicate rows): C skips the comparison entirely
|
|
325
|
+
# when the candidate *is* the object, so this assignment cannot be beaten.
|
|
326
|
+
labels[medoids] = np.arange(len(medoids))
|
|
327
|
+
dist = np.sqrt(dsq[np.arange(x.shape[0]), labels])
|
|
328
|
+
return labels, dist
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _selec(x: np.ndarray, medoids: np.ndarray) -> tuple[float, np.ndarray]:
|
|
332
|
+
"""Score one sub-sample's medoids over the whole dataset (``selec``).
|
|
333
|
+
|
|
334
|
+
Returns ``(zb, medoids_reordered)`` where ``zb`` is the summed distance —
|
|
335
|
+
lower is better — and the medoids come back permuted into the order their
|
|
336
|
+
clusters are *first encountered* scanning objects in order, which is the
|
|
337
|
+
ordering that ends up deciding R's cluster numbering.
|
|
338
|
+
"""
|
|
339
|
+
labels, dist = _assign(x, medoids)
|
|
340
|
+
# C accumulates `*zb += dnull` one object at a time, and zb is compared across
|
|
341
|
+
# sub-samples with `>`, so the rounding has to match or near-ties between two
|
|
342
|
+
# samples resolve the wrong way. np.sum() reorders (pairwise summation);
|
|
343
|
+
# cumsum is sequential by definition, and its last element is the same float
|
|
344
|
+
# C arrives at.
|
|
345
|
+
zb = float(np.cumsum(dist)[-1]) if dist.size else 0.0
|
|
346
|
+
|
|
347
|
+
# C tracks first appearance in new[] and permutes nr[] by it. Every cluster
|
|
348
|
+
# is guaranteed non-empty (each medoid holds itself), so new[] always fills.
|
|
349
|
+
seen: list[int] = []
|
|
350
|
+
flagged = np.zeros(len(medoids), dtype=bool)
|
|
351
|
+
for lab in labels:
|
|
352
|
+
if not flagged[lab]:
|
|
353
|
+
flagged[lab] = True
|
|
354
|
+
seen.append(int(lab))
|
|
355
|
+
if len(seen) == len(medoids):
|
|
356
|
+
break
|
|
357
|
+
return zb, medoids[np.array(seen, dtype=int)]
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
# ----------------------------------------------------------------------
|
|
361
|
+
# The driver (clara.c :: cl_clara)
|
|
362
|
+
# ----------------------------------------------------------------------
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _draw_sample(
|
|
366
|
+
rng: _Randm, n: int, n_sam: int, kk: int, nrx: np.ndarray, kall: bool,
|
|
367
|
+
jran: int, lrg_sam: bool,
|
|
368
|
+
) -> np.ndarray:
|
|
369
|
+
"""Draw one sub-sample's indices, sorted ascending (0-based).
|
|
370
|
+
|
|
371
|
+
Reproduces the index-drawing block of ``cl_clara``. Once a valid sample has
|
|
372
|
+
been seen, and unless we are sampling more than half the data, the running
|
|
373
|
+
best medoids are seeded into the sample first and the remainder drawn around
|
|
374
|
+
them — Kaufman & Rousseeuw's "each sub-dataset is forced to contain the
|
|
375
|
+
medoids obtained from the best sub-dataset until then".
|
|
376
|
+
|
|
377
|
+
``n_sam`` is the number of indices to *draw*, which is the complement's size
|
|
378
|
+
when ``lrg_sam``; the caller inverts.
|
|
379
|
+
"""
|
|
380
|
+
def draw() -> int:
|
|
381
|
+
# C: rand_k = 1 + (int)(rnn * randm(&nrun)), clamped; 1-based there.
|
|
382
|
+
rand_k = int(n * rng())
|
|
383
|
+
return min(rand_k, n - 1)
|
|
384
|
+
|
|
385
|
+
nsel: list[int] = []
|
|
386
|
+
|
|
387
|
+
# nunfs (the count of samples abandoned to missing data) is always 0 here --
|
|
388
|
+
# we reject NaNs up front -- so C's `nunfs + 1 != jran` reduces to `jran != 1`.
|
|
389
|
+
if kall and jran != 1 and not lrg_sam:
|
|
390
|
+
nsel = sorted(int(v) for v in nrx[:kk])
|
|
391
|
+
else:
|
|
392
|
+
while True:
|
|
393
|
+
rand_k = draw()
|
|
394
|
+
if kall and rand_k in nrx[:kk]:
|
|
395
|
+
continue
|
|
396
|
+
break
|
|
397
|
+
nsel.append(rand_k)
|
|
398
|
+
if len(nsel) == n_sam:
|
|
399
|
+
return np.array(nsel, dtype=int)
|
|
400
|
+
|
|
401
|
+
# C runs this as a do-while, so it always adds at least one index.
|
|
402
|
+
while True:
|
|
403
|
+
while True:
|
|
404
|
+
rand_k = draw()
|
|
405
|
+
if kall and lrg_sam and rand_k in nrx[:kk]:
|
|
406
|
+
continue
|
|
407
|
+
# C walks nsel linearly for the first entry >= rand_k and inserts
|
|
408
|
+
# there, keeping it sorted; a redraw on an exact hit. Same position.
|
|
409
|
+
pos = bisect.bisect_left(nsel, rand_k)
|
|
410
|
+
if pos < len(nsel) and nsel[pos] == rand_k:
|
|
411
|
+
continue # already sampled -- redraw
|
|
412
|
+
nsel.insert(pos, rand_k)
|
|
413
|
+
break
|
|
414
|
+
if len(nsel) >= n_sam:
|
|
415
|
+
return np.array(nsel, dtype=int)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _cl_clara(x: np.ndarray, kk: int, nran: int, nsam: int) -> np.ndarray:
|
|
419
|
+
"""The resampling loop of ``cl_clara``; returns 0-based labels."""
|
|
420
|
+
if not np.isfinite(x).all():
|
|
421
|
+
# C's clara handles NA by tracking per-column missing codes and can bail
|
|
422
|
+
# out with jstop; none of that is ported, so refuse rather than silently
|
|
423
|
+
# disagree with R.
|
|
424
|
+
raise ValueError(
|
|
425
|
+
"clara() does not support missing or non-finite values; "
|
|
426
|
+
"R's clara handles them by a separate code path that is not ported."
|
|
427
|
+
)
|
|
428
|
+
|
|
429
|
+
n = x.shape[0]
|
|
430
|
+
nsamb = nsam * 2
|
|
431
|
+
full_sample = n == nsam
|
|
432
|
+
lrg_sam = n < nsamb # sampling more than half -- draw the complement instead
|
|
433
|
+
n_sam = n - nsam if lrg_sam else nsam
|
|
434
|
+
|
|
435
|
+
rng = _Randm()
|
|
436
|
+
kall = False
|
|
437
|
+
zba = -1.0
|
|
438
|
+
nrx = np.zeros(kk, dtype=int)
|
|
439
|
+
|
|
440
|
+
for jran in range(1, nran + 1):
|
|
441
|
+
if full_sample:
|
|
442
|
+
nsel = np.arange(n)
|
|
443
|
+
else:
|
|
444
|
+
nsel = _draw_sample(rng, n, n_sam, kk, nrx, kall, jran, lrg_sam)
|
|
445
|
+
if lrg_sam:
|
|
446
|
+
# We hold the *unsampled* complement; invert it.
|
|
447
|
+
mask = np.ones(n, dtype=bool)
|
|
448
|
+
mask[nsel] = False
|
|
449
|
+
nsel = np.flatnonzero(mask)
|
|
450
|
+
|
|
451
|
+
dys = _pairwise_dys(x, nsel)
|
|
452
|
+
# C maxes over dys[1..n_dys], skipping the permanently-zero dys[0]; with
|
|
453
|
+
# non-negative distances that is just the matrix maximum.
|
|
454
|
+
s = float(dys.max()) if dys.size else 0.0
|
|
455
|
+
|
|
456
|
+
nrepr = _bswap2(kk, dys, s)
|
|
457
|
+
zb, medoids = _selec(x, nsel[nrepr])
|
|
458
|
+
|
|
459
|
+
if not kall or zba > zb: # first proper sample, or a new best
|
|
460
|
+
kall = True
|
|
461
|
+
zba = zb
|
|
462
|
+
nrx = medoids
|
|
463
|
+
|
|
464
|
+
if full_sample:
|
|
465
|
+
break # further samples would be identical
|
|
466
|
+
|
|
467
|
+
# resul(): assign the entire dataset to the winning medoids.
|
|
468
|
+
labels, _ = _assign(x, nrx)
|
|
469
|
+
return labels
|