imste 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {imste-0.2.1 → imste-0.3.0}/PKG-INFO +17 -19
- {imste-0.2.1 → imste-0.3.0}/README.md +15 -18
- {imste-0.2.1 → imste-0.3.0}/imste/_api.py +8 -2
- {imste-0.2.1 → imste-0.3.0}/imste/_estimator_graph.py +43 -9
- imste-0.2.1/imste/class_graph.py → imste-0.3.0/imste/_visualization_class_graph.py +4 -204
- imste-0.3.0/imste/class_graph.py +243 -0
- {imste-0.2.1 → imste-0.3.0}/imste/visualization.py +2 -0
- {imste-0.2.1 → imste-0.3.0}/imste.egg-info/PKG-INFO +17 -19
- {imste-0.2.1 → imste-0.3.0}/imste.egg-info/SOURCES.txt +3 -1
- {imste-0.2.1 → imste-0.3.0}/imste.egg-info/requires.txt +1 -0
- {imste-0.2.1 → imste-0.3.0}/pyproject.toml +2 -2
- imste-0.3.0/tests/test_visualization.py +68 -0
- {imste-0.2.1 → imste-0.3.0}/LICENSE +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/__init__.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_estimator.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_estimator_utils.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_gravity.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_mean_shift.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_multilevel.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_optimizer.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_optimizer_mps.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_optimizer_torch.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_progress.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_visualization_class_geometry.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_visualization_core.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/_visualization_interactive.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/inductive.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/losses.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/mst/__init__.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/mst/famst.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/mst/graph.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste/mst/prim.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste.egg-info/dependency_links.txt +0 -0
- {imste-0.2.1 → imste-0.3.0}/imste.egg-info/top_level.txt +0 -0
- {imste-0.2.1 → imste-0.3.0}/setup.cfg +0 -0
- {imste-0.2.1 → imste-0.3.0}/tests/test_arxiv_notebook_helpers.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/tests/test_estimator.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/tests/test_famst.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/tests/test_multilevel.py +0 -0
- {imste-0.2.1 → imste-0.3.0}/tests/test_optimizer.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: imste
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Iterated Minimum Spanning Tree Embedding (IMSTE)
|
|
5
5
|
Author: Fabrizio Costa
|
|
6
6
|
License-Expression: MIT
|
|
@@ -33,6 +33,7 @@ Provides-Extra: test
|
|
|
33
33
|
Requires-Dist: pytest>=7; extra == "test"
|
|
34
34
|
Requires-Dist: pandas>=2.0; extra == "test"
|
|
35
35
|
Requires-Dist: torch>=2.0; extra == "test"
|
|
36
|
+
Requires-Dist: matplotlib>=3.7; extra == "test"
|
|
36
37
|
Provides-Extra: notebook
|
|
37
38
|
Requires-Dist: matplotlib>=3.7; extra == "notebook"
|
|
38
39
|
Requires-Dist: pandas>=2.0; extra == "notebook"
|
|
@@ -47,6 +48,8 @@ Dynamic: license-file
|
|
|
47
48
|
|
|
48
49
|
# IMSTE
|
|
49
50
|
|
|
51
|
+
[](https://doi.org/10.5281/zenodo.24307529)
|
|
52
|
+
|
|
50
53
|
**Iterated Minimum Spanning Tree Embedding (IMSTE)** is a scikit-learn
|
|
51
54
|
transformer for learning low-dimensional coordinates from numeric data. It
|
|
52
55
|
builds a weighted graph from ranked minimum spanning trees, then
|
|
@@ -89,9 +92,6 @@ python -m pip install "imste[notebook]"
|
|
|
89
92
|
|
|
90
93
|
From a source checkout, use `python -m pip install ".[notebook]"` instead.
|
|
91
94
|
|
|
92
|
-
For maintainers preparing a release, see the
|
|
93
|
-
[release guide](docs/releasing.md).
|
|
94
|
-
|
|
95
95
|
Both estimators accept dense arrays and SciPy CSR matrices. The graph builders
|
|
96
96
|
keep CSR inputs sparse. `InductiveIMSTE` densifies only each MLP mini-batch.
|
|
97
97
|
The default FAMST graph builder uses RINND for dense Euclidean inputs and
|
|
@@ -161,7 +161,10 @@ X = StandardScaler().fit_transform(digits.data)
|
|
|
161
161
|
distance_matrix = cdist(X, X, metric="euclidean")
|
|
162
162
|
distance_graph = csr_matrix(distance_matrix)
|
|
163
163
|
|
|
164
|
-
|
|
164
|
+
# First fit IMSTE to X, then use that layout to initialize the graph fit.
|
|
165
|
+
graph_embedding = IteratedMinimumSpanningTreeEmbedder().fit_graph_transform(
|
|
166
|
+
distance_graph, X=X
|
|
167
|
+
)
|
|
165
168
|
|
|
166
169
|
plt.scatter(graph_embedding[:, 0], graph_embedding[:, 1], c=digits.target, cmap="tab10", s=10)
|
|
167
170
|
```
|
|
@@ -353,7 +356,7 @@ directly; they use `embedding_` or the inductive model's `reference_embedding_`.
|
|
|
353
356
|
Raw coordinate arrays remain supported.
|
|
354
357
|
|
|
355
358
|
For class-level graph overlays across one or more embeddings, use
|
|
356
|
-
`imste.
|
|
359
|
+
`imste.visualization.plot_class_mixture_mst`. It fits full-covariance Gaussian
|
|
357
360
|
mixtures per class in each 2D embedding, connects their component centers with
|
|
358
361
|
iterated MSTs, and maps symmetric cross-component posterior probability to
|
|
359
362
|
edge opacity. For each edge, the two endpoint Gaussian densities form an
|
|
@@ -442,7 +445,9 @@ score into `[-1, 1]`. This preserves the relative scale between PCA axes. Set
|
|
|
442
445
|
`pca_init_scaling="none"` to preserve raw PCA score magnitudes, or
|
|
443
446
|
`pca_init=False` to use the seeded random initialization instead.
|
|
444
447
|
`fit_graph()` uses the same PCA initialization when `init_features` is supplied;
|
|
445
|
-
otherwise it uses random initialization.
|
|
448
|
+
otherwise it uses random initialization. `fit_graph_transform(graph, X=X)` first
|
|
449
|
+
fits IMSTE to `X` and uses that layout to initialize the graph fit. Pass
|
|
450
|
+
`init_coordinates` with shape
|
|
446
451
|
`(n_samples, n_components)` to start from explicit coordinates instead:
|
|
447
452
|
|
|
448
453
|
```python
|
|
@@ -517,24 +522,17 @@ optimization by passing an `IteratedMinimumSpanningTreeEmbedder` as `embedder`.
|
|
|
517
522
|
|
|
518
523
|
## Notebooks
|
|
519
524
|
|
|
520
|
-
Active experiment notebooks use concise dataset-based names. Dataset
|
|
525
|
+
Active experiment notebooks use concise dataset-based names. Dataset setup notebooks live in `notebooks/dataset_setup/`.
|
|
521
526
|
|
|
522
527
|
- [synthetic_exp](notebooks/synthetic_exp.ipynb): compare IMSTE and UMAP on controlled geometries.
|
|
523
|
-
- [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories.
|
|
528
|
+
- [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories. Prepare its local inputs with [prepare_biological_datasets](notebooks/dataset_setup/prepare_biological_datasets.ipynb).
|
|
524
529
|
- [arxiv_exp](notebooks/arxiv_exp.ipynb): compare embeddings of the prepared arXiv sample.
|
|
525
|
-
- [
|
|
530
|
+
- [cifar100_exp](notebooks/cifar100_exp.ipynb): compare CIFAR-100 image embeddings with superclass hues and fine-class shades.
|
|
526
531
|
- [interactive](notebooks/interactive.ipynb): interactively refine a balanced arXiv layout with gravity followed by mean shift.
|
|
527
532
|
- [mnist_exp](notebooks/mnist_exp.ipynb): compare MNIST, Fashion-MNIST, and Kuzushiji-MNIST.
|
|
528
|
-
- [tabula_sapiens_exp](notebooks/tabula_sapiens_exp.ipynb): compare tissue and cell-type colorings on shared layouts.
|
|
529
|
-
|
|
530
|
-
Prepare source data with [the arXiv preparation notebook](data/arxiv/prepare_dataset.ipynb) and [the CIFAR-10 preparation notebook](data/cifar10/prepare_dataset.ipynb).
|
|
533
|
+
- [tabula_sapiens_exp](notebooks/tabula_sapiens_exp.ipynb): load prepared Tabula Sapiens features and compare tissue and cell-type colorings on shared layouts.
|
|
531
534
|
|
|
532
|
-
|
|
533
|
-
[notebooks/archive](notebooks/archive/). This includes the [interactive 2D
|
|
534
|
-
embedding](notebooks/archive/digits_2d_interactive.ipynb), [interactive 3D
|
|
535
|
-
embedding](notebooks/archive/digits_3d_interactive.ipynb), [parameter
|
|
536
|
-
sweep](notebooks/archive/digits_parameter_sweep.ipynb), and [MST scaling
|
|
537
|
-
benchmark](notebooks/archive/mst_scaling_benchmark.ipynb).
|
|
535
|
+
Prepare source data with the [arXiv](notebooks/dataset_setup/prepare_arxiv_dataset.ipynb), [CIFAR-100](notebooks/dataset_setup/prepare_cifar100_dataset.ipynb), and [Tabula Sapiens](notebooks/dataset_setup/prepare_tabula_sapiens_dataset.ipynb) setup notebooks.
|
|
538
536
|
|
|
539
537
|
Python helper modules used by the notebooks are in [notebooks/src](notebooks/src/).
|
|
540
538
|
For the full method description and equations, see the
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# IMSTE
|
|
2
2
|
|
|
3
|
+
[](https://doi.org/10.5281/zenodo.24307529)
|
|
4
|
+
|
|
3
5
|
**Iterated Minimum Spanning Tree Embedding (IMSTE)** is a scikit-learn
|
|
4
6
|
transformer for learning low-dimensional coordinates from numeric data. It
|
|
5
7
|
builds a weighted graph from ranked minimum spanning trees, then
|
|
@@ -42,9 +44,6 @@ python -m pip install "imste[notebook]"
|
|
|
42
44
|
|
|
43
45
|
From a source checkout, use `python -m pip install ".[notebook]"` instead.
|
|
44
46
|
|
|
45
|
-
For maintainers preparing a release, see the
|
|
46
|
-
[release guide](docs/releasing.md).
|
|
47
|
-
|
|
48
47
|
Both estimators accept dense arrays and SciPy CSR matrices. The graph builders
|
|
49
48
|
keep CSR inputs sparse. `InductiveIMSTE` densifies only each MLP mini-batch.
|
|
50
49
|
The default FAMST graph builder uses RINND for dense Euclidean inputs and
|
|
@@ -114,7 +113,10 @@ X = StandardScaler().fit_transform(digits.data)
|
|
|
114
113
|
distance_matrix = cdist(X, X, metric="euclidean")
|
|
115
114
|
distance_graph = csr_matrix(distance_matrix)
|
|
116
115
|
|
|
117
|
-
|
|
116
|
+
# First fit IMSTE to X, then use that layout to initialize the graph fit.
|
|
117
|
+
graph_embedding = IteratedMinimumSpanningTreeEmbedder().fit_graph_transform(
|
|
118
|
+
distance_graph, X=X
|
|
119
|
+
)
|
|
118
120
|
|
|
119
121
|
plt.scatter(graph_embedding[:, 0], graph_embedding[:, 1], c=digits.target, cmap="tab10", s=10)
|
|
120
122
|
```
|
|
@@ -306,7 +308,7 @@ directly; they use `embedding_` or the inductive model's `reference_embedding_`.
|
|
|
306
308
|
Raw coordinate arrays remain supported.
|
|
307
309
|
|
|
308
310
|
For class-level graph overlays across one or more embeddings, use
|
|
309
|
-
`imste.
|
|
311
|
+
`imste.visualization.plot_class_mixture_mst`. It fits full-covariance Gaussian
|
|
310
312
|
mixtures per class in each 2D embedding, connects their component centers with
|
|
311
313
|
iterated MSTs, and maps symmetric cross-component posterior probability to
|
|
312
314
|
edge opacity. For each edge, the two endpoint Gaussian densities form an
|
|
@@ -395,7 +397,9 @@ score into `[-1, 1]`. This preserves the relative scale between PCA axes. Set
|
|
|
395
397
|
`pca_init_scaling="none"` to preserve raw PCA score magnitudes, or
|
|
396
398
|
`pca_init=False` to use the seeded random initialization instead.
|
|
397
399
|
`fit_graph()` uses the same PCA initialization when `init_features` is supplied;
|
|
398
|
-
otherwise it uses random initialization.
|
|
400
|
+
otherwise it uses random initialization. `fit_graph_transform(graph, X=X)` first
|
|
401
|
+
fits IMSTE to `X` and uses that layout to initialize the graph fit. Pass
|
|
402
|
+
`init_coordinates` with shape
|
|
399
403
|
`(n_samples, n_components)` to start from explicit coordinates instead:
|
|
400
404
|
|
|
401
405
|
```python
|
|
@@ -470,24 +474,17 @@ optimization by passing an `IteratedMinimumSpanningTreeEmbedder` as `embedder`.
|
|
|
470
474
|
|
|
471
475
|
## Notebooks
|
|
472
476
|
|
|
473
|
-
Active experiment notebooks use concise dataset-based names. Dataset
|
|
477
|
+
Active experiment notebooks use concise dataset-based names. Dataset setup notebooks live in `notebooks/dataset_setup/`.
|
|
474
478
|
|
|
475
479
|
- [synthetic_exp](notebooks/synthetic_exp.ipynb): compare IMSTE and UMAP on controlled geometries.
|
|
476
|
-
- [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories.
|
|
480
|
+
- [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories. Prepare its local inputs with [prepare_biological_datasets](notebooks/dataset_setup/prepare_biological_datasets.ipynb).
|
|
477
481
|
- [arxiv_exp](notebooks/arxiv_exp.ipynb): compare embeddings of the prepared arXiv sample.
|
|
478
|
-
- [
|
|
482
|
+
- [cifar100_exp](notebooks/cifar100_exp.ipynb): compare CIFAR-100 image embeddings with superclass hues and fine-class shades.
|
|
479
483
|
- [interactive](notebooks/interactive.ipynb): interactively refine a balanced arXiv layout with gravity followed by mean shift.
|
|
480
484
|
- [mnist_exp](notebooks/mnist_exp.ipynb): compare MNIST, Fashion-MNIST, and Kuzushiji-MNIST.
|
|
481
|
-
- [tabula_sapiens_exp](notebooks/tabula_sapiens_exp.ipynb): compare tissue and cell-type colorings on shared layouts.
|
|
482
|
-
|
|
483
|
-
Prepare source data with [the arXiv preparation notebook](data/arxiv/prepare_dataset.ipynb) and [the CIFAR-10 preparation notebook](data/cifar10/prepare_dataset.ipynb).
|
|
485
|
+
- [tabula_sapiens_exp](notebooks/tabula_sapiens_exp.ipynb): load prepared Tabula Sapiens features and compare tissue and cell-type colorings on shared layouts.
|
|
484
486
|
|
|
485
|
-
|
|
486
|
-
[notebooks/archive](notebooks/archive/). This includes the [interactive 2D
|
|
487
|
-
embedding](notebooks/archive/digits_2d_interactive.ipynb), [interactive 3D
|
|
488
|
-
embedding](notebooks/archive/digits_3d_interactive.ipynb), [parameter
|
|
489
|
-
sweep](notebooks/archive/digits_parameter_sweep.ipynb), and [MST scaling
|
|
490
|
-
benchmark](notebooks/archive/mst_scaling_benchmark.ipynb).
|
|
487
|
+
Prepare source data with the [arXiv](notebooks/dataset_setup/prepare_arxiv_dataset.ipynb), [CIFAR-100](notebooks/dataset_setup/prepare_cifar100_dataset.ipynb), and [Tabula Sapiens](notebooks/dataset_setup/prepare_tabula_sapiens_dataset.ipynb) setup notebooks.
|
|
491
488
|
|
|
492
489
|
Python helper modules used by the notebooks are in [notebooks/src](notebooks/src/).
|
|
493
490
|
For the full method description and equations, see the
|
|
@@ -13,16 +13,22 @@ from ._estimator import IteratedMinimumSpanningTreeEmbedder
|
|
|
13
13
|
def fit_graph_transform(
|
|
14
14
|
distance_graph: spmatrix,
|
|
15
15
|
*,
|
|
16
|
+
X: object | None = None,
|
|
16
17
|
init_features: object | None = None,
|
|
17
18
|
**estimator_params: Any,
|
|
18
19
|
) -> np.ndarray:
|
|
19
20
|
"""Fit IMSTE from a sparse distance graph and return its embedding.
|
|
20
21
|
|
|
21
|
-
``
|
|
22
|
-
|
|
22
|
+
If ``X`` is supplied, IMSTE first embeds those features, then uses that
|
|
23
|
+
layout to initialize optimization over ``distance_graph``. The graph
|
|
24
|
+
determines the final optimization objective. Alternatively,
|
|
25
|
+
``init_features`` supplies features for standardized-PCA initialization.
|
|
26
|
+
These two initialization options cannot be combined. Remaining keyword
|
|
27
|
+
arguments configure the estimator.
|
|
23
28
|
"""
|
|
24
29
|
estimator = IteratedMinimumSpanningTreeEmbedder(**estimator_params)
|
|
25
30
|
return estimator.fit_graph_transform(
|
|
26
31
|
distance_graph,
|
|
32
|
+
X=X,
|
|
27
33
|
init_features=init_features,
|
|
28
34
|
)
|
|
@@ -7,6 +7,7 @@ import time
|
|
|
7
7
|
import numpy as np
|
|
8
8
|
from scipy import sparse
|
|
9
9
|
from scipy.sparse import csr_matrix, issparse
|
|
10
|
+
from sklearn.base import clone
|
|
10
11
|
from sklearn.utils.validation import check_array, check_is_fitted
|
|
11
12
|
|
|
12
13
|
from .mst import iterated_graph_edges
|
|
@@ -19,6 +20,7 @@ class _EstimatorGraphMethods:
|
|
|
19
20
|
distance_graph: sparse.spmatrix,
|
|
20
21
|
*,
|
|
21
22
|
init_features: object | None = None,
|
|
23
|
+
init_coordinates: object | None = None,
|
|
22
24
|
) -> "IteratedMinimumSpanningTreeEmbedder":
|
|
23
25
|
"""Fit directly from a sparse graph whose stored values are distances.
|
|
24
26
|
|
|
@@ -33,12 +35,13 @@ class _EstimatorGraphMethods:
|
|
|
33
35
|
``init_features`` optionally supplies the sample-by-feature matrix used
|
|
34
36
|
for standardized PCA initialization. ``init_coordinates`` can instead
|
|
35
37
|
supply a complete initial embedding with one row per graph node and one
|
|
36
|
-
column per output dimension; it takes precedence over PCA
|
|
37
|
-
|
|
38
|
-
graph
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
column per output dimension; it takes precedence over PCA and the
|
|
39
|
+
estimator's ``init_coordinates`` parameter. All rows must follow the
|
|
40
|
+
graph's order. Initialization affects coordinates only; graph edges
|
|
41
|
+
still determine the optimization objective. Without either form of
|
|
42
|
+
initialization, the seeded random start is used. The fitted coordinates
|
|
43
|
+
are available as ``embedding_``; feature-based ``transform`` is
|
|
44
|
+
unavailable after ``fit_graph``.
|
|
42
45
|
"""
|
|
43
46
|
fit_started = time.perf_counter()
|
|
44
47
|
options = self._validated_fit_options()
|
|
@@ -74,6 +77,9 @@ class _EstimatorGraphMethods:
|
|
|
74
77
|
if options["gravity_enabled"] and initialization_features is not None:
|
|
75
78
|
if issparse(initialization_features):
|
|
76
79
|
raise ValueError("gravity refinement currently requires dense features.")
|
|
80
|
+
graph_initial_coordinates = (
|
|
81
|
+
self.init_coordinates if init_coordinates is None else init_coordinates
|
|
82
|
+
)
|
|
77
83
|
self._activate_progress_reporting()
|
|
78
84
|
graph_started = time.perf_counter()
|
|
79
85
|
edges, tree_ranks = iterated_graph_edges(
|
|
@@ -97,7 +103,7 @@ class _EstimatorGraphMethods:
|
|
|
97
103
|
resolved_device=options["resolved_device"],
|
|
98
104
|
schedule_by_rank=True,
|
|
99
105
|
initialization_features=initialization_features,
|
|
100
|
-
initial_coordinates=
|
|
106
|
+
initial_coordinates=graph_initial_coordinates,
|
|
101
107
|
**{
|
|
102
108
|
name: value
|
|
103
109
|
for name, value in options.items()
|
|
@@ -182,10 +188,38 @@ class _EstimatorGraphMethods:
|
|
|
182
188
|
self,
|
|
183
189
|
distance_graph: sparse.spmatrix,
|
|
184
190
|
*,
|
|
191
|
+
X: object | None = None,
|
|
185
192
|
init_features: object | None = None,
|
|
186
193
|
) -> np.ndarray:
|
|
187
|
-
"""Fit from a sparse distance graph and return the embedding array.
|
|
188
|
-
|
|
194
|
+
"""Fit from a sparse distance graph and return the embedding array.
|
|
195
|
+
|
|
196
|
+
If ``X`` is supplied, fit IMSTE to ``X`` first and use that embedding
|
|
197
|
+
to initialize optimization over ``distance_graph``. The graph remains
|
|
198
|
+
the objective for the final fit. ``init_features`` retains the direct
|
|
199
|
+
standardized-PCA initialization behavior of :meth:`fit_graph` and
|
|
200
|
+
cannot be combined with ``X``.
|
|
201
|
+
"""
|
|
202
|
+
if X is None:
|
|
203
|
+
self.fit_graph(distance_graph, init_features=init_features)
|
|
204
|
+
return self.embedding_.copy()
|
|
205
|
+
if init_features is not None:
|
|
206
|
+
raise ValueError("Pass either X or init_features, not both.")
|
|
207
|
+
if (
|
|
208
|
+
not sparse.issparse(distance_graph)
|
|
209
|
+
or distance_graph.ndim != 2
|
|
210
|
+
or distance_graph.shape[0] != distance_graph.shape[1]
|
|
211
|
+
):
|
|
212
|
+
raise ValueError("distance_graph must be a square SciPy sparse matrix.")
|
|
213
|
+
if distance_graph.shape[0] < 2:
|
|
214
|
+
raise ValueError("distance_graph must contain at least two samples.")
|
|
215
|
+
X_checked = check_array(
|
|
216
|
+
X, dtype=np.float64, ensure_2d=True, accept_sparse="csr"
|
|
217
|
+
)
|
|
218
|
+
if X_checked.shape[0] != distance_graph.shape[0]:
|
|
219
|
+
raise ValueError("X must have one row per distance_graph node.")
|
|
220
|
+
feature_estimator = clone(self)
|
|
221
|
+
initialization = feature_estimator.fit_transform(X_checked)
|
|
222
|
+
self.fit_graph(distance_graph, init_coordinates=initialization)
|
|
189
223
|
return self.embedding_.copy()
|
|
190
224
|
|
|
191
225
|
def transform(self, X: object) -> np.ndarray:
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Reusable
|
|
1
|
+
"""Reusable class-mixture plots for labeled low-dimensional embeddings."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -10,114 +10,11 @@ import pandas as pd
|
|
|
10
10
|
from matplotlib.colors import BoundaryNorm, ListedColormap
|
|
11
11
|
from matplotlib.lines import Line2D
|
|
12
12
|
from matplotlib.patches import FancyArrowPatch
|
|
13
|
-
from scipy.special import expit
|
|
14
|
-
from scipy.stats import multivariate_normal
|
|
15
|
-
from sklearn.mixture import GaussianMixture
|
|
16
|
-
from imste.mst.prim import iterated_prim_edges
|
|
17
13
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def _fit_class_mixture_nodes(
|
|
22
|
-
embedding: np.ndarray,
|
|
23
|
-
labels: np.ndarray,
|
|
24
|
-
*,
|
|
25
|
-
max_components: int,
|
|
26
|
-
min_component_size: int,
|
|
27
|
-
random_state: int,
|
|
28
|
-
) -> tuple[list[dict[str, object]], np.ndarray, np.ndarray]:
|
|
29
|
-
"""Fit full-covariance GMMs per class and return component nodes and labels."""
|
|
30
|
-
classes = np.unique(labels)
|
|
31
|
-
nodes: list[dict[str, object]] = []
|
|
32
|
-
sample_node_ids = np.full(len(labels), -1, dtype=np.intp)
|
|
33
|
-
class_codes = np.empty(len(labels), dtype=np.intp)
|
|
34
|
-
|
|
35
|
-
for class_code, class_label in enumerate(classes):
|
|
36
|
-
sample_indices = np.flatnonzero(labels == class_label)
|
|
37
|
-
class_codes[sample_indices] = class_code
|
|
38
|
-
n_components = min(
|
|
39
|
-
max_components,
|
|
40
|
-
max(1, len(sample_indices) // min_component_size),
|
|
41
|
-
)
|
|
42
|
-
while True:
|
|
43
|
-
mixture = GaussianMixture(
|
|
44
|
-
n_components=n_components,
|
|
45
|
-
covariance_type="full",
|
|
46
|
-
reg_covar=1e-5,
|
|
47
|
-
n_init=3,
|
|
48
|
-
random_state=random_state + class_code,
|
|
49
|
-
).fit(embedding[sample_indices])
|
|
50
|
-
component_ids = mixture.predict(embedding[sample_indices])
|
|
51
|
-
counts = np.bincount(component_ids, minlength=n_components)
|
|
52
|
-
if n_components == 1 or counts.min() >= min_component_size:
|
|
53
|
-
break
|
|
54
|
-
n_components -= 1
|
|
55
|
-
|
|
56
|
-
for component_id in range(n_components):
|
|
57
|
-
members = sample_indices[component_ids == component_id]
|
|
58
|
-
node_id = len(nodes)
|
|
59
|
-
sample_node_ids[members] = node_id
|
|
60
|
-
nodes.append(
|
|
61
|
-
{
|
|
62
|
-
"class_label": class_label,
|
|
63
|
-
"class_code": class_code,
|
|
64
|
-
"component_id": component_id,
|
|
65
|
-
"position": mixture.means_[component_id].copy(),
|
|
66
|
-
"covariance": mixture.covariances_[component_id].copy(),
|
|
67
|
-
"mixture_weight": float(mixture.weights_[component_id]),
|
|
68
|
-
"sample_indices": members,
|
|
69
|
-
"sample_coordinates": embedding[members].copy(),
|
|
70
|
-
"n_samples": len(members),
|
|
71
|
-
}
|
|
72
|
-
)
|
|
73
|
-
|
|
74
|
-
if np.any(sample_node_ids < 0):
|
|
75
|
-
raise RuntimeError("Some samples were not assigned to a class-mixture component.")
|
|
76
|
-
return nodes, sample_node_ids, class_codes
|
|
14
|
+
from .class_graph import _edge_component_overlap, _fit_class_mixture_nodes
|
|
15
|
+
from .mst.prim import iterated_prim_edges
|
|
77
16
|
|
|
78
|
-
|
|
79
|
-
def _edge_component_overlap(
|
|
80
|
-
nodes: list[dict[str, object]],
|
|
81
|
-
left: int,
|
|
82
|
-
right: int,
|
|
83
|
-
) -> tuple[float, float, float]:
|
|
84
|
-
"""Return symmetric cross-component posterior overlap in 2D."""
|
|
85
|
-
node_a, node_b = nodes[left], nodes[right]
|
|
86
|
-
if node_a["class_label"] == node_b["class_label"]:
|
|
87
|
-
raise ValueError("Same-class edges should be filtered before scoring.")
|
|
88
|
-
indices_a = np.asarray(node_a["sample_indices"], dtype=np.intp)
|
|
89
|
-
indices_b = np.asarray(node_b["sample_indices"], dtype=np.intp)
|
|
90
|
-
covariance_a = np.asarray(node_a["covariance"])
|
|
91
|
-
covariance_b = np.asarray(node_b["covariance"])
|
|
92
|
-
log_density_a_on_b = multivariate_normal.logpdf(
|
|
93
|
-
np.asarray(node_b["sample_coordinates"]),
|
|
94
|
-
mean=np.asarray(node_a["position"]),
|
|
95
|
-
cov=covariance_a,
|
|
96
|
-
allow_singular=True,
|
|
97
|
-
)
|
|
98
|
-
log_density_b_on_b = multivariate_normal.logpdf(
|
|
99
|
-
np.asarray(node_b["sample_coordinates"]),
|
|
100
|
-
mean=np.asarray(node_b["position"]),
|
|
101
|
-
cov=covariance_b,
|
|
102
|
-
allow_singular=True,
|
|
103
|
-
)
|
|
104
|
-
log_density_a_on_a = multivariate_normal.logpdf(
|
|
105
|
-
np.asarray(node_a["sample_coordinates"]),
|
|
106
|
-
mean=np.asarray(node_a["position"]),
|
|
107
|
-
cov=covariance_a,
|
|
108
|
-
allow_singular=True,
|
|
109
|
-
)
|
|
110
|
-
log_density_b_on_a = multivariate_normal.logpdf(
|
|
111
|
-
np.asarray(node_a["sample_coordinates"]),
|
|
112
|
-
mean=np.asarray(node_b["position"]),
|
|
113
|
-
cov=covariance_b,
|
|
114
|
-
allow_singular=True,
|
|
115
|
-
)
|
|
116
|
-
# Equal priors make this an endpoint distinguishability score, independent
|
|
117
|
-
# of the relative number of samples in each component.
|
|
118
|
-
prob_a_to_b = float(np.mean(expit(log_density_a_on_b - log_density_b_on_b)))
|
|
119
|
-
prob_b_to_a = float(np.mean(expit(log_density_b_on_a - log_density_a_on_a)))
|
|
120
|
-
return prob_a_to_b, prob_b_to_a, (prob_a_to_b + prob_b_to_a) / 2.0
|
|
17
|
+
NODE_EDGE_WIDTH = 1.1
|
|
121
18
|
|
|
122
19
|
|
|
123
20
|
def _overlap_to_alpha(score: float, faint: float, opaque: float) -> float:
|
|
@@ -126,103 +23,6 @@ def _overlap_to_alpha(score: float, faint: float, opaque: float) -> float:
|
|
|
126
23
|
return float(faint + confusion * (opaque - faint))
|
|
127
24
|
|
|
128
25
|
|
|
129
|
-
def compare_class_graph_relations(
|
|
130
|
-
edge_scores: pd.DataFrame,
|
|
131
|
-
*,
|
|
132
|
-
method_a: str = "IMSTE",
|
|
133
|
-
method_b: str = "UMAP",
|
|
134
|
-
dataset: object | None = None,
|
|
135
|
-
) -> pd.DataFrame:
|
|
136
|
-
"""Rank class-pair MST relations unique to either embedding.
|
|
137
|
-
|
|
138
|
-
Component-level edges are collapsed to unordered class pairs because the
|
|
139
|
-
GMM components are fit independently in each embedding. A relation is
|
|
140
|
-
considered present when at least one cross-class component edge connects
|
|
141
|
-
that class pair. Duplicate appearances across iterated MSTs count once.
|
|
142
|
-
``edge_error`` is the mean chance-normalized symmetric cross-assignment
|
|
143
|
-
probability across the distinct component edges for that class pair; the
|
|
144
|
-
table also reports the maximum edge error and component-level details.
|
|
145
|
-
|
|
146
|
-
Rows are sorted by descending edge error separately for relations unique
|
|
147
|
-
to ``method_a`` and ``method_b``. A chance-level cross-assignment
|
|
148
|
-
probability of 0.5 maps to edge error 1.0; perfect separation maps to 0.
|
|
149
|
-
"""
|
|
150
|
-
required = {
|
|
151
|
-
"embedding", "class_a", "component_a", "class_b", "component_b",
|
|
152
|
-
"symmetric_component_overlap",
|
|
153
|
-
}
|
|
154
|
-
missing = required.difference(edge_scores.columns)
|
|
155
|
-
if missing:
|
|
156
|
-
raise ValueError(f"edge_scores is missing columns: {sorted(missing)}")
|
|
157
|
-
if method_a == method_b:
|
|
158
|
-
raise ValueError("method_a and method_b must be different embedding names.")
|
|
159
|
-
frame = edge_scores.copy()
|
|
160
|
-
if dataset is not None:
|
|
161
|
-
if "dataset" not in frame:
|
|
162
|
-
raise ValueError("dataset was specified but edge_scores has no dataset column.")
|
|
163
|
-
frame = frame.loc[frame["dataset"] == dataset]
|
|
164
|
-
frame = frame.loc[frame["embedding"].isin([method_a, method_b])]
|
|
165
|
-
|
|
166
|
-
def label_key(value: object) -> tuple[str, str]:
|
|
167
|
-
return type(value).__name__, str(value)
|
|
168
|
-
|
|
169
|
-
# method -> unordered label pair -> component edge -> edge error
|
|
170
|
-
relations: dict[str, dict[tuple[object, object], dict[tuple[object, object], float]]] = {
|
|
171
|
-
method_a: {}, method_b: {}
|
|
172
|
-
}
|
|
173
|
-
for row in frame.to_dict("records"):
|
|
174
|
-
left, right = row["class_a"], row["class_b"]
|
|
175
|
-
if label_key(left) <= label_key(right):
|
|
176
|
-
class_pair = (left, right)
|
|
177
|
-
component_pair = (row["component_a"], row["component_b"])
|
|
178
|
-
else:
|
|
179
|
-
class_pair = (right, left)
|
|
180
|
-
component_pair = (row["component_b"], row["component_a"])
|
|
181
|
-
error = float(np.clip(row["symmetric_component_overlap"] / 0.5, 0.0, 1.0))
|
|
182
|
-
method = row["embedding"]
|
|
183
|
-
relation_edges = relations[method].setdefault(class_pair, {})
|
|
184
|
-
# The same component edge can appear in multiple iterated MSTs.
|
|
185
|
-
relation_edges[component_pair] = max(error, relation_edges.get(component_pair, 0.0))
|
|
186
|
-
|
|
187
|
-
rows: list[dict[str, object]] = []
|
|
188
|
-
for source, target in ((method_a, method_b), (method_b, method_a)):
|
|
189
|
-
unique_pairs = set(relations[source]).difference(relations[target])
|
|
190
|
-
ranked = []
|
|
191
|
-
for class_pair in unique_pairs:
|
|
192
|
-
component_errors = relations[source][class_pair]
|
|
193
|
-
errors = list(component_errors.values())
|
|
194
|
-
ranked.append(
|
|
195
|
-
{
|
|
196
|
-
"class_a": class_pair[0],
|
|
197
|
-
"class_b": class_pair[1],
|
|
198
|
-
"present_in": source,
|
|
199
|
-
"missing_from": target,
|
|
200
|
-
"edge_error": float(np.mean(errors)),
|
|
201
|
-
"max_component_edge_error": float(np.max(errors)),
|
|
202
|
-
"n_component_edges": len(errors),
|
|
203
|
-
"component_edges": tuple(
|
|
204
|
-
(component_a, component_b, component_errors[(component_a, component_b)])
|
|
205
|
-
for component_a, component_b in sorted(
|
|
206
|
-
component_errors, key=lambda pair: (str(pair[0]), str(pair[1]))
|
|
207
|
-
)
|
|
208
|
-
),
|
|
209
|
-
}
|
|
210
|
-
)
|
|
211
|
-
ranked.sort(key=lambda row: (-row["edge_error"], -row["max_component_edge_error"],
|
|
212
|
-
label_key(row["class_a"]), label_key(row["class_b"])))
|
|
213
|
-
for rank, row in enumerate(ranked, start=1):
|
|
214
|
-
rows.append({"rank": rank, **row})
|
|
215
|
-
|
|
216
|
-
return pd.DataFrame(
|
|
217
|
-
rows,
|
|
218
|
-
columns=[
|
|
219
|
-
"rank", "class_a", "class_b", "present_in", "missing_from",
|
|
220
|
-
"edge_error", "max_component_edge_error", "n_component_edges",
|
|
221
|
-
"component_edges",
|
|
222
|
-
],
|
|
223
|
-
)
|
|
224
|
-
|
|
225
|
-
|
|
226
26
|
def plot_class_mixture_mst(
|
|
227
27
|
X: np.ndarray,
|
|
228
28
|
labels: object,
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
"""Analysis helpers for class-mixture minimum-spanning-tree relations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pandas as pd
|
|
9
|
+
from scipy.special import expit
|
|
10
|
+
from scipy.stats import multivariate_normal
|
|
11
|
+
from sklearn.mixture import GaussianMixture
|
|
12
|
+
|
|
13
|
+
def _fit_class_mixture_nodes(
|
|
14
|
+
embedding: np.ndarray,
|
|
15
|
+
labels: np.ndarray,
|
|
16
|
+
*,
|
|
17
|
+
max_components: int,
|
|
18
|
+
min_component_size: int,
|
|
19
|
+
random_state: int,
|
|
20
|
+
) -> tuple[list[dict[str, object]], np.ndarray, np.ndarray]:
|
|
21
|
+
"""Fit full-covariance GMMs per class and return component nodes and labels."""
|
|
22
|
+
classes = np.unique(labels)
|
|
23
|
+
nodes: list[dict[str, object]] = []
|
|
24
|
+
sample_node_ids = np.full(len(labels), -1, dtype=np.intp)
|
|
25
|
+
class_codes = np.empty(len(labels), dtype=np.intp)
|
|
26
|
+
|
|
27
|
+
for class_code, class_label in enumerate(classes):
|
|
28
|
+
sample_indices = np.flatnonzero(labels == class_label)
|
|
29
|
+
class_codes[sample_indices] = class_code
|
|
30
|
+
n_components = min(
|
|
31
|
+
max_components,
|
|
32
|
+
max(1, len(sample_indices) // min_component_size),
|
|
33
|
+
)
|
|
34
|
+
while True:
|
|
35
|
+
mixture = GaussianMixture(
|
|
36
|
+
n_components=n_components,
|
|
37
|
+
covariance_type="full",
|
|
38
|
+
reg_covar=1e-5,
|
|
39
|
+
n_init=3,
|
|
40
|
+
random_state=random_state + class_code,
|
|
41
|
+
).fit(embedding[sample_indices])
|
|
42
|
+
component_ids = mixture.predict(embedding[sample_indices])
|
|
43
|
+
counts = np.bincount(component_ids, minlength=n_components)
|
|
44
|
+
if n_components == 1 or counts.min() >= min_component_size:
|
|
45
|
+
break
|
|
46
|
+
n_components -= 1
|
|
47
|
+
|
|
48
|
+
for component_id in range(n_components):
|
|
49
|
+
members = sample_indices[component_ids == component_id]
|
|
50
|
+
node_id = len(nodes)
|
|
51
|
+
sample_node_ids[members] = node_id
|
|
52
|
+
nodes.append(
|
|
53
|
+
{
|
|
54
|
+
"class_label": class_label,
|
|
55
|
+
"class_code": class_code,
|
|
56
|
+
"component_id": component_id,
|
|
57
|
+
"position": mixture.means_[component_id].copy(),
|
|
58
|
+
"covariance": mixture.covariances_[component_id].copy(),
|
|
59
|
+
"mixture_weight": float(mixture.weights_[component_id]),
|
|
60
|
+
"sample_indices": members,
|
|
61
|
+
"sample_coordinates": embedding[members].copy(),
|
|
62
|
+
"n_samples": len(members),
|
|
63
|
+
}
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
if np.any(sample_node_ids < 0):
|
|
67
|
+
raise RuntimeError("Some samples were not assigned to a class-mixture component.")
|
|
68
|
+
return nodes, sample_node_ids, class_codes
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _edge_component_overlap(
|
|
72
|
+
nodes: list[dict[str, object]],
|
|
73
|
+
left: int,
|
|
74
|
+
right: int,
|
|
75
|
+
) -> tuple[float, float, float]:
|
|
76
|
+
"""Return symmetric cross-component posterior overlap in 2D."""
|
|
77
|
+
node_a, node_b = nodes[left], nodes[right]
|
|
78
|
+
if node_a["class_label"] == node_b["class_label"]:
|
|
79
|
+
raise ValueError("Same-class edges should be filtered before scoring.")
|
|
80
|
+
indices_a = np.asarray(node_a["sample_indices"], dtype=np.intp)
|
|
81
|
+
indices_b = np.asarray(node_b["sample_indices"], dtype=np.intp)
|
|
82
|
+
covariance_a = np.asarray(node_a["covariance"])
|
|
83
|
+
covariance_b = np.asarray(node_b["covariance"])
|
|
84
|
+
log_density_a_on_b = multivariate_normal.logpdf(
|
|
85
|
+
np.asarray(node_b["sample_coordinates"]),
|
|
86
|
+
mean=np.asarray(node_a["position"]),
|
|
87
|
+
cov=covariance_a,
|
|
88
|
+
allow_singular=True,
|
|
89
|
+
)
|
|
90
|
+
log_density_b_on_b = multivariate_normal.logpdf(
|
|
91
|
+
np.asarray(node_b["sample_coordinates"]),
|
|
92
|
+
mean=np.asarray(node_b["position"]),
|
|
93
|
+
cov=covariance_b,
|
|
94
|
+
allow_singular=True,
|
|
95
|
+
)
|
|
96
|
+
log_density_a_on_a = multivariate_normal.logpdf(
|
|
97
|
+
np.asarray(node_a["sample_coordinates"]),
|
|
98
|
+
mean=np.asarray(node_a["position"]),
|
|
99
|
+
cov=covariance_a,
|
|
100
|
+
allow_singular=True,
|
|
101
|
+
)
|
|
102
|
+
log_density_b_on_a = multivariate_normal.logpdf(
|
|
103
|
+
np.asarray(node_a["sample_coordinates"]),
|
|
104
|
+
mean=np.asarray(node_b["position"]),
|
|
105
|
+
cov=covariance_b,
|
|
106
|
+
allow_singular=True,
|
|
107
|
+
)
|
|
108
|
+
# Equal priors make this an endpoint distinguishability score, independent
|
|
109
|
+
# of the relative number of samples in each component.
|
|
110
|
+
prob_a_to_b = float(np.mean(expit(log_density_a_on_b - log_density_b_on_b)))
|
|
111
|
+
prob_b_to_a = float(np.mean(expit(log_density_b_on_a - log_density_a_on_a)))
|
|
112
|
+
return prob_a_to_b, prob_b_to_a, (prob_a_to_b + prob_b_to_a) / 2.0
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def compare_class_graph_relations(
|
|
116
|
+
edge_scores: pd.DataFrame,
|
|
117
|
+
*,
|
|
118
|
+
method_a: str = "IMSTE",
|
|
119
|
+
method_b: str = "UMAP",
|
|
120
|
+
dataset: object | None = None,
|
|
121
|
+
) -> pd.DataFrame:
|
|
122
|
+
"""Rank class-pair MST relations unique to either embedding.
|
|
123
|
+
|
|
124
|
+
Component-level edges are collapsed to unordered class pairs because the
|
|
125
|
+
GMM components are fit independently in each embedding. A relation is
|
|
126
|
+
considered present when at least one cross-class component edge connects
|
|
127
|
+
that class pair. Duplicate appearances across iterated MSTs count once.
|
|
128
|
+
``edge_error`` is the mean chance-normalized symmetric cross-assignment
|
|
129
|
+
probability across the distinct component edges for that class pair; the
|
|
130
|
+
table also reports the maximum edge error and component-level details.
|
|
131
|
+
|
|
132
|
+
Rows are sorted by descending edge error separately for relations unique
|
|
133
|
+
to ``method_a`` and ``method_b``. A chance-level cross-assignment
|
|
134
|
+
probability of 0.5 maps to edge error 1.0; perfect separation maps to 0.
|
|
135
|
+
"""
|
|
136
|
+
required = {
|
|
137
|
+
"embedding", "class_a", "component_a", "class_b", "component_b",
|
|
138
|
+
"symmetric_component_overlap",
|
|
139
|
+
}
|
|
140
|
+
missing = required.difference(edge_scores.columns)
|
|
141
|
+
if missing:
|
|
142
|
+
raise ValueError(f"edge_scores is missing columns: {sorted(missing)}")
|
|
143
|
+
if method_a == method_b:
|
|
144
|
+
raise ValueError("method_a and method_b must be different embedding names.")
|
|
145
|
+
frame = edge_scores.copy()
|
|
146
|
+
if dataset is not None:
|
|
147
|
+
if "dataset" not in frame:
|
|
148
|
+
raise ValueError("dataset was specified but edge_scores has no dataset column.")
|
|
149
|
+
frame = frame.loc[frame["dataset"] == dataset]
|
|
150
|
+
frame = frame.loc[frame["embedding"].isin([method_a, method_b])]
|
|
151
|
+
|
|
152
|
+
def label_key(value: object) -> tuple[str, str]:
|
|
153
|
+
return type(value).__name__, str(value)
|
|
154
|
+
|
|
155
|
+
# method -> unordered label pair -> component edge -> edge error
|
|
156
|
+
relations: dict[str, dict[tuple[object, object], dict[tuple[object, object], float]]] = {
|
|
157
|
+
method_a: {}, method_b: {}
|
|
158
|
+
}
|
|
159
|
+
for row in frame.to_dict("records"):
|
|
160
|
+
left, right = row["class_a"], row["class_b"]
|
|
161
|
+
if label_key(left) <= label_key(right):
|
|
162
|
+
class_pair = (left, right)
|
|
163
|
+
component_pair = (row["component_a"], row["component_b"])
|
|
164
|
+
else:
|
|
165
|
+
class_pair = (right, left)
|
|
166
|
+
component_pair = (row["component_b"], row["component_a"])
|
|
167
|
+
error = float(np.clip(row["symmetric_component_overlap"] / 0.5, 0.0, 1.0))
|
|
168
|
+
method = row["embedding"]
|
|
169
|
+
relation_edges = relations[method].setdefault(class_pair, {})
|
|
170
|
+
# The same component edge can appear in multiple iterated MSTs.
|
|
171
|
+
relation_edges[component_pair] = max(error, relation_edges.get(component_pair, 0.0))
|
|
172
|
+
|
|
173
|
+
rows: list[dict[str, object]] = []
|
|
174
|
+
for source, target in ((method_a, method_b), (method_b, method_a)):
|
|
175
|
+
unique_pairs = set(relations[source]).difference(relations[target])
|
|
176
|
+
ranked = []
|
|
177
|
+
for class_pair in unique_pairs:
|
|
178
|
+
component_errors = relations[source][class_pair]
|
|
179
|
+
errors = list(component_errors.values())
|
|
180
|
+
ranked.append(
|
|
181
|
+
{
|
|
182
|
+
"class_a": class_pair[0],
|
|
183
|
+
"class_b": class_pair[1],
|
|
184
|
+
"present_in": source,
|
|
185
|
+
"missing_from": target,
|
|
186
|
+
"edge_error": float(np.mean(errors)),
|
|
187
|
+
"max_component_edge_error": float(np.max(errors)),
|
|
188
|
+
"n_component_edges": len(errors),
|
|
189
|
+
"component_edges": tuple(
|
|
190
|
+
(component_a, component_b, component_errors[(component_a, component_b)])
|
|
191
|
+
for component_a, component_b in sorted(
|
|
192
|
+
component_errors, key=lambda pair: (str(pair[0]), str(pair[1]))
|
|
193
|
+
)
|
|
194
|
+
),
|
|
195
|
+
}
|
|
196
|
+
)
|
|
197
|
+
ranked.sort(key=lambda row: (-row["edge_error"], -row["max_component_edge_error"],
|
|
198
|
+
label_key(row["class_a"]), label_key(row["class_b"])))
|
|
199
|
+
for rank, row in enumerate(ranked, start=1):
|
|
200
|
+
rows.append({"rank": rank, **row})
|
|
201
|
+
|
|
202
|
+
return pd.DataFrame(
|
|
203
|
+
rows,
|
|
204
|
+
columns=[
|
|
205
|
+
"rank", "class_a", "class_b", "present_in", "missing_from",
|
|
206
|
+
"edge_error", "max_component_edge_error", "n_component_edges",
|
|
207
|
+
"component_edges",
|
|
208
|
+
],
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def plot_class_mixture_mst(
|
|
213
|
+
X: np.ndarray,
|
|
214
|
+
labels: object,
|
|
215
|
+
embeddings: Mapping[str, object],
|
|
216
|
+
*,
|
|
217
|
+
title: str | None = None,
|
|
218
|
+
n_msts: int = 2,
|
|
219
|
+
max_components_per_class: int = 3,
|
|
220
|
+
min_component_size: int = 20,
|
|
221
|
+
point_alpha: float = 0.4,
|
|
222
|
+
min_edge_alpha: float = 0.2,
|
|
223
|
+
max_edge_alpha: float = 0.9,
|
|
224
|
+
random_state: int = 42,
|
|
225
|
+
class_colors: Mapping[object, object] | None = None,
|
|
226
|
+
):
|
|
227
|
+
"""Backward-compatible import for the plotting API in ``imste.visualization``."""
|
|
228
|
+
from .visualization import plot_class_mixture_mst as plotter
|
|
229
|
+
|
|
230
|
+
return plotter(
|
|
231
|
+
X,
|
|
232
|
+
labels,
|
|
233
|
+
embeddings,
|
|
234
|
+
title=title,
|
|
235
|
+
n_msts=n_msts,
|
|
236
|
+
max_components_per_class=max_components_per_class,
|
|
237
|
+
min_component_size=min_component_size,
|
|
238
|
+
point_alpha=point_alpha,
|
|
239
|
+
min_edge_alpha=min_edge_alpha,
|
|
240
|
+
max_edge_alpha=max_edge_alpha,
|
|
241
|
+
random_state=random_state,
|
|
242
|
+
class_colors=class_colors,
|
|
243
|
+
)
|
|
@@ -19,10 +19,12 @@ from ._visualization_class_geometry import (
|
|
|
19
19
|
plot_ellipses,
|
|
20
20
|
plot_knn_union,
|
|
21
21
|
)
|
|
22
|
+
from ._visualization_class_graph import plot_class_mixture_mst
|
|
22
23
|
|
|
23
24
|
__all__ = [
|
|
24
25
|
"cross_validate_embedding_knn",
|
|
25
26
|
"interactive_force",
|
|
27
|
+
"plot_class_mixture_mst",
|
|
26
28
|
"plot_density_parent_tree",
|
|
27
29
|
"plot_ellipses",
|
|
28
30
|
"plot_embedding",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: imste
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Iterated Minimum Spanning Tree Embedding (IMSTE)
|
|
5
5
|
Author: Fabrizio Costa
|
|
6
6
|
License-Expression: MIT
|
|
@@ -33,6 +33,7 @@ Provides-Extra: test
|
|
|
33
33
|
Requires-Dist: pytest>=7; extra == "test"
|
|
34
34
|
Requires-Dist: pandas>=2.0; extra == "test"
|
|
35
35
|
Requires-Dist: torch>=2.0; extra == "test"
|
|
36
|
+
Requires-Dist: matplotlib>=3.7; extra == "test"
|
|
36
37
|
Provides-Extra: notebook
|
|
37
38
|
Requires-Dist: matplotlib>=3.7; extra == "notebook"
|
|
38
39
|
Requires-Dist: pandas>=2.0; extra == "notebook"
|
|
@@ -47,6 +48,8 @@ Dynamic: license-file
|
|
|
47
48
|
|
|
48
49
|
# IMSTE
|
|
49
50
|
|
|
51
|
+
[](https://doi.org/10.5281/zenodo.24307529)
|
|
52
|
+
|
|
50
53
|
**Iterated Minimum Spanning Tree Embedding (IMSTE)** is a scikit-learn
|
|
51
54
|
transformer for learning low-dimensional coordinates from numeric data. It
|
|
52
55
|
builds a weighted graph from ranked minimum spanning trees, then
|
|
@@ -89,9 +92,6 @@ python -m pip install "imste[notebook]"
|
|
|
89
92
|
|
|
90
93
|
From a source checkout, use `python -m pip install ".[notebook]"` instead.
|
|
91
94
|
|
|
92
|
-
For maintainers preparing a release, see the
|
|
93
|
-
[release guide](docs/releasing.md).
|
|
94
|
-
|
|
95
95
|
Both estimators accept dense arrays and SciPy CSR matrices. The graph builders
|
|
96
96
|
keep CSR inputs sparse. `InductiveIMSTE` densifies only each MLP mini-batch.
|
|
97
97
|
The default FAMST graph builder uses RINND for dense Euclidean inputs and
|
|
@@ -161,7 +161,10 @@ X = StandardScaler().fit_transform(digits.data)
|
|
|
161
161
|
distance_matrix = cdist(X, X, metric="euclidean")
|
|
162
162
|
distance_graph = csr_matrix(distance_matrix)
|
|
163
163
|
|
|
164
|
-
|
|
164
|
+
# First fit IMSTE to X, then use that layout to initialize the graph fit.
|
|
165
|
+
graph_embedding = IteratedMinimumSpanningTreeEmbedder().fit_graph_transform(
|
|
166
|
+
distance_graph, X=X
|
|
167
|
+
)
|
|
165
168
|
|
|
166
169
|
plt.scatter(graph_embedding[:, 0], graph_embedding[:, 1], c=digits.target, cmap="tab10", s=10)
|
|
167
170
|
```
|
|
@@ -353,7 +356,7 @@ directly; they use `embedding_` or the inductive model's `reference_embedding_`.
|
|
|
353
356
|
Raw coordinate arrays remain supported.
|
|
354
357
|
|
|
355
358
|
For class-level graph overlays across one or more embeddings, use
|
|
356
|
-
`imste.
|
|
359
|
+
`imste.visualization.plot_class_mixture_mst`. It fits full-covariance Gaussian
|
|
357
360
|
mixtures per class in each 2D embedding, connects their component centers with
|
|
358
361
|
iterated MSTs, and maps symmetric cross-component posterior probability to
|
|
359
362
|
edge opacity. For each edge, the two endpoint Gaussian densities form an
|
|
@@ -442,7 +445,9 @@ score into `[-1, 1]`. This preserves the relative scale between PCA axes. Set
|
|
|
442
445
|
`pca_init_scaling="none"` to preserve raw PCA score magnitudes, or
|
|
443
446
|
`pca_init=False` to use the seeded random initialization instead.
|
|
444
447
|
`fit_graph()` uses the same PCA initialization when `init_features` is supplied;
|
|
445
|
-
otherwise it uses random initialization.
|
|
448
|
+
otherwise it uses random initialization. `fit_graph_transform(graph, X=X)` first
|
|
449
|
+
fits IMSTE to `X` and uses that layout to initialize the graph fit. Pass
|
|
450
|
+
`init_coordinates` with shape
|
|
446
451
|
`(n_samples, n_components)` to start from explicit coordinates instead:
|
|
447
452
|
|
|
448
453
|
```python
|
|
@@ -517,24 +522,17 @@ optimization by passing an `IteratedMinimumSpanningTreeEmbedder` as `embedder`.
|
|
|
517
522
|
|
|
518
523
|
## Notebooks
|
|
519
524
|
|
|
520
|
-
Active experiment notebooks use concise dataset-based names. Dataset
|
|
525
|
+
Active experiment notebooks use concise dataset-based names. Dataset setup notebooks live in `notebooks/dataset_setup/`.
|
|
521
526
|
|
|
522
527
|
- [synthetic_exp](notebooks/synthetic_exp.ipynb): compare IMSTE and UMAP on controlled geometries.
|
|
523
|
-
- [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories.
|
|
528
|
+
- [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories. Prepare its local inputs with [prepare_biological_datasets](notebooks/dataset_setup/prepare_biological_datasets.ipynb).
|
|
524
529
|
- [arxiv_exp](notebooks/arxiv_exp.ipynb): compare embeddings of the prepared arXiv sample.
|
|
525
|
-
- [
|
|
530
|
+
- [cifar100_exp](notebooks/cifar100_exp.ipynb): compare CIFAR-100 image embeddings with superclass hues and fine-class shades.
|
|
526
531
|
- [interactive](notebooks/interactive.ipynb): interactively refine a balanced arXiv layout with gravity followed by mean shift.
|
|
527
532
|
- [mnist_exp](notebooks/mnist_exp.ipynb): compare MNIST, Fashion-MNIST, and Kuzushiji-MNIST.
|
|
528
|
-
- [tabula_sapiens_exp](notebooks/tabula_sapiens_exp.ipynb): compare tissue and cell-type colorings on shared layouts.
|
|
529
|
-
|
|
530
|
-
Prepare source data with [the arXiv preparation notebook](data/arxiv/prepare_dataset.ipynb) and [the CIFAR-10 preparation notebook](data/cifar10/prepare_dataset.ipynb).
|
|
533
|
+
- [tabula_sapiens_exp](notebooks/tabula_sapiens_exp.ipynb): load prepared Tabula Sapiens features and compare tissue and cell-type colorings on shared layouts.
|
|
531
534
|
|
|
532
|
-
|
|
533
|
-
[notebooks/archive](notebooks/archive/). This includes the [interactive 2D
|
|
534
|
-
embedding](notebooks/archive/digits_2d_interactive.ipynb), [interactive 3D
|
|
535
|
-
embedding](notebooks/archive/digits_3d_interactive.ipynb), [parameter
|
|
536
|
-
sweep](notebooks/archive/digits_parameter_sweep.ipynb), and [MST scaling
|
|
537
|
-
benchmark](notebooks/archive/mst_scaling_benchmark.ipynb).
|
|
535
|
+
Prepare source data with the [arXiv](notebooks/dataset_setup/prepare_arxiv_dataset.ipynb), [CIFAR-100](notebooks/dataset_setup/prepare_cifar100_dataset.ipynb), and [Tabula Sapiens](notebooks/dataset_setup/prepare_tabula_sapiens_dataset.ipynb) setup notebooks.
|
|
538
536
|
|
|
539
537
|
Python helper modules used by the notebooks are in [notebooks/src](notebooks/src/).
|
|
540
538
|
For the full method description and equations, see the
|
|
@@ -14,6 +14,7 @@ imste/_optimizer_mps.py
|
|
|
14
14
|
imste/_optimizer_torch.py
|
|
15
15
|
imste/_progress.py
|
|
16
16
|
imste/_visualization_class_geometry.py
|
|
17
|
+
imste/_visualization_class_graph.py
|
|
17
18
|
imste/_visualization_core.py
|
|
18
19
|
imste/_visualization_interactive.py
|
|
19
20
|
imste/class_graph.py
|
|
@@ -33,4 +34,5 @@ tests/test_arxiv_notebook_helpers.py
|
|
|
33
34
|
tests/test_estimator.py
|
|
34
35
|
tests/test_famst.py
|
|
35
36
|
tests/test_multilevel.py
|
|
36
|
-
tests/test_optimizer.py
|
|
37
|
+
tests/test_optimizer.py
|
|
38
|
+
tests/test_visualization.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "imste"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Iterated Minimum Spanning Tree Embedding (IMSTE)"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -40,7 +40,7 @@ Documentation = "https://github.com/fabriziocosta/IteratedMinimumSpanningTree/tr
|
|
|
40
40
|
|
|
41
41
|
[project.optional-dependencies]
|
|
42
42
|
gpu = ["torch>=2.0"]
|
|
43
|
-
test = ["pytest>=7", "pandas>=2.0", "torch>=2.0"]
|
|
43
|
+
test = ["pytest>=7", "pandas>=2.0", "torch>=2.0", "matplotlib>=3.7"]
|
|
44
44
|
notebook = [
|
|
45
45
|
"matplotlib>=3.7",
|
|
46
46
|
"pandas>=2.0",
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import matplotlib
|
|
2
|
+
import numpy as np
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
matplotlib.use("Agg", force=True)
|
|
7
|
+
import matplotlib.pyplot as plt
|
|
8
|
+
|
|
9
|
+
from imste.class_graph import plot_class_mixture_mst as legacy_plot_class_mixture_mst
|
|
10
|
+
from imste.visualization import plot_class_mixture_mst, plot_embedding
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_plot_embedding_accepts_series_and_returns_figure_and_axes():
|
|
14
|
+
coordinates = np.array([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]])
|
|
15
|
+
labels = pd.Series(["a", "a", "b", "b"], name="class")
|
|
16
|
+
|
|
17
|
+
figure, axes = plot_embedding(coordinates, labels)
|
|
18
|
+
|
|
19
|
+
assert figure is axes[0, 0].figure
|
|
20
|
+
assert axes.shape == (1, 1)
|
|
21
|
+
assert axes[0, 0].get_title() == "IMSTE colored by class"
|
|
22
|
+
plt.close(figure)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_plot_embedding_rejects_label_count_mismatch():
|
|
26
|
+
with pytest.raises(ValueError, match="obs rows must match"):
|
|
27
|
+
plot_embedding(np.zeros((3, 2)), ["a", "b"])
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_class_mixture_plot_is_in_visualization_api_and_legacy_import_still_works(
|
|
31
|
+
monkeypatch,
|
|
32
|
+
):
|
|
33
|
+
expected = object()
|
|
34
|
+
monkeypatch.setattr("imste.visualization.plot_class_mixture_mst", lambda *a, **k: expected)
|
|
35
|
+
|
|
36
|
+
result = legacy_plot_class_mixture_mst(
|
|
37
|
+
np.zeros((2, 1)), np.array(["a", "b"]), {"IMSTE": np.zeros((2, 2))}
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
assert result is expected
|
|
41
|
+
assert callable(plot_class_mixture_mst)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_class_mixture_plot_returns_axes_and_edge_scores():
|
|
45
|
+
rng = np.random.default_rng(2)
|
|
46
|
+
labels = np.repeat(["left", "right"], 24)
|
|
47
|
+
embedding = np.vstack(
|
|
48
|
+
[rng.normal(loc=(-2.0, 0.0), scale=0.25, size=(24, 2)),
|
|
49
|
+
rng.normal(loc=(2.0, 0.0), scale=0.25, size=(24, 2))]
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
figure, axes, edge_scores = plot_class_mixture_mst(
|
|
53
|
+
embedding,
|
|
54
|
+
labels,
|
|
55
|
+
{"IMSTE": embedding, "UMAP": embedding + np.array([0.1, -0.1])},
|
|
56
|
+
n_msts=1,
|
|
57
|
+
max_components_per_class=1,
|
|
58
|
+
min_component_size=5,
|
|
59
|
+
random_state=0,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
assert len(axes) == 2
|
|
63
|
+
assert {"embedding", "class_a", "class_b", "symmetric_component_overlap"}.issubset(
|
|
64
|
+
edge_scores.columns
|
|
65
|
+
)
|
|
66
|
+
assert set(edge_scores["embedding"]) == {"IMSTE", "UMAP"}
|
|
67
|
+
assert len(edge_scores) == 2
|
|
68
|
+
plt.close(figure)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|