imste 0.2.2__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. {imste-0.2.2 → imste-0.3.0}/PKG-INFO +8 -15
  2. {imste-0.2.2 → imste-0.3.0}/README.md +6 -14
  3. imste-0.2.2/imste/class_graph.py → imste-0.3.0/imste/_visualization_class_graph.py +4 -204
  4. imste-0.3.0/imste/class_graph.py +243 -0
  5. {imste-0.2.2 → imste-0.3.0}/imste/visualization.py +2 -0
  6. {imste-0.2.2 → imste-0.3.0}/imste.egg-info/PKG-INFO +8 -15
  7. {imste-0.2.2 → imste-0.3.0}/imste.egg-info/SOURCES.txt +3 -1
  8. {imste-0.2.2 → imste-0.3.0}/imste.egg-info/requires.txt +1 -0
  9. {imste-0.2.2 → imste-0.3.0}/pyproject.toml +2 -2
  10. imste-0.3.0/tests/test_visualization.py +68 -0
  11. {imste-0.2.2 → imste-0.3.0}/LICENSE +0 -0
  12. {imste-0.2.2 → imste-0.3.0}/imste/__init__.py +0 -0
  13. {imste-0.2.2 → imste-0.3.0}/imste/_api.py +0 -0
  14. {imste-0.2.2 → imste-0.3.0}/imste/_estimator.py +0 -0
  15. {imste-0.2.2 → imste-0.3.0}/imste/_estimator_graph.py +0 -0
  16. {imste-0.2.2 → imste-0.3.0}/imste/_estimator_utils.py +0 -0
  17. {imste-0.2.2 → imste-0.3.0}/imste/_gravity.py +0 -0
  18. {imste-0.2.2 → imste-0.3.0}/imste/_mean_shift.py +0 -0
  19. {imste-0.2.2 → imste-0.3.0}/imste/_multilevel.py +0 -0
  20. {imste-0.2.2 → imste-0.3.0}/imste/_optimizer.py +0 -0
  21. {imste-0.2.2 → imste-0.3.0}/imste/_optimizer_mps.py +0 -0
  22. {imste-0.2.2 → imste-0.3.0}/imste/_optimizer_torch.py +0 -0
  23. {imste-0.2.2 → imste-0.3.0}/imste/_progress.py +0 -0
  24. {imste-0.2.2 → imste-0.3.0}/imste/_visualization_class_geometry.py +0 -0
  25. {imste-0.2.2 → imste-0.3.0}/imste/_visualization_core.py +0 -0
  26. {imste-0.2.2 → imste-0.3.0}/imste/_visualization_interactive.py +0 -0
  27. {imste-0.2.2 → imste-0.3.0}/imste/inductive.py +0 -0
  28. {imste-0.2.2 → imste-0.3.0}/imste/losses.py +0 -0
  29. {imste-0.2.2 → imste-0.3.0}/imste/mst/__init__.py +0 -0
  30. {imste-0.2.2 → imste-0.3.0}/imste/mst/famst.py +0 -0
  31. {imste-0.2.2 → imste-0.3.0}/imste/mst/graph.py +0 -0
  32. {imste-0.2.2 → imste-0.3.0}/imste/mst/prim.py +0 -0
  33. {imste-0.2.2 → imste-0.3.0}/imste.egg-info/dependency_links.txt +0 -0
  34. {imste-0.2.2 → imste-0.3.0}/imste.egg-info/top_level.txt +0 -0
  35. {imste-0.2.2 → imste-0.3.0}/setup.cfg +0 -0
  36. {imste-0.2.2 → imste-0.3.0}/tests/test_arxiv_notebook_helpers.py +0 -0
  37. {imste-0.2.2 → imste-0.3.0}/tests/test_estimator.py +0 -0
  38. {imste-0.2.2 → imste-0.3.0}/tests/test_famst.py +0 -0
  39. {imste-0.2.2 → imste-0.3.0}/tests/test_multilevel.py +0 -0
  40. {imste-0.2.2 → imste-0.3.0}/tests/test_optimizer.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: imste
3
- Version: 0.2.2
3
+ Version: 0.3.0
4
4
  Summary: Iterated Minimum Spanning Tree Embedding (IMSTE)
5
5
  Author: Fabrizio Costa
6
6
  License-Expression: MIT
@@ -33,6 +33,7 @@ Provides-Extra: test
33
33
  Requires-Dist: pytest>=7; extra == "test"
34
34
  Requires-Dist: pandas>=2.0; extra == "test"
35
35
  Requires-Dist: torch>=2.0; extra == "test"
36
+ Requires-Dist: matplotlib>=3.7; extra == "test"
36
37
  Provides-Extra: notebook
37
38
  Requires-Dist: matplotlib>=3.7; extra == "notebook"
38
39
  Requires-Dist: pandas>=2.0; extra == "notebook"
@@ -47,6 +48,8 @@ Dynamic: license-file
47
48
 
48
49
  # IMSTE
49
50
 
51
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.24307529.svg)](https://doi.org/10.5281/zenodo.24307529)
52
+
50
53
  **Iterated Minimum Spanning Tree Embedding (IMSTE)** is a scikit-learn
51
54
  transformer for learning low-dimensional coordinates from numeric data. It
52
55
  builds a weighted graph from ranked minimum spanning trees, then
@@ -89,9 +92,6 @@ python -m pip install "imste[notebook]"
89
92
 
90
93
  From a source checkout, use `python -m pip install ".[notebook]"` instead.
91
94
 
92
- For maintainers preparing a release, see the
93
- [release guide](docs/releasing.md).
94
-
95
95
  Both estimators accept dense arrays and SciPy CSR matrices. The graph builders
96
96
  keep CSR inputs sparse. `InductiveIMSTE` densifies only each MLP mini-batch.
97
97
  The default FAMST graph builder uses RINND for dense Euclidean inputs and
@@ -356,7 +356,7 @@ directly; they use `embedding_` or the inductive model's `reference_embedding_`.
356
356
  Raw coordinate arrays remain supported.
357
357
 
358
358
  For class-level graph overlays across one or more embeddings, use
359
- `imste.class_graph.plot_class_mixture_mst`. It fits full-covariance Gaussian
359
+ `imste.visualization.plot_class_mixture_mst`. It fits full-covariance Gaussian
360
360
  mixtures per class in each 2D embedding, connects their component centers with
361
361
  iterated MSTs, and maps symmetric cross-component posterior probability to
362
362
  edge opacity. For each edge, the two endpoint Gaussian densities form an
@@ -525,21 +525,14 @@ optimization by passing an `IteratedMinimumSpanningTreeEmbedder` as `embedder`.
525
525
  Active experiment notebooks use concise dataset-based names. Dataset setup notebooks live in `notebooks/dataset_setup/`.
526
526
 
527
527
  - [synthetic_exp](notebooks/synthetic_exp.ipynb): compare IMSTE and UMAP on controlled geometries.
528
- - [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories.
528
+ - [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories. Prepare its local inputs with [prepare_biological_datasets](notebooks/dataset_setup/prepare_biological_datasets.ipynb).
529
529
  - [arxiv_exp](notebooks/arxiv_exp.ipynb): compare embeddings of the prepared arXiv sample.
530
- - [cifar10_exp](notebooks/cifar10_exp.ipynb): compare CIFAR-10 image-vector embeddings.
530
+ - [cifar100_exp](notebooks/cifar100_exp.ipynb): compare CIFAR-100 image embeddings with superclass hues and fine-class shades.
531
531
  - [interactive](notebooks/interactive.ipynb): interactively refine a balanced arXiv layout with gravity followed by mean shift.
532
532
  - [mnist_exp](notebooks/mnist_exp.ipynb): compare MNIST, Fashion-MNIST, and Kuzushiji-MNIST.
533
533
  - [tabula_sapiens_exp](notebooks/tabula_sapiens_exp.ipynb): load prepared Tabula Sapiens features and compare tissue and cell-type colorings on shared layouts.
534
534
 
535
- Prepare source data with the [arXiv](notebooks/dataset_setup/prepare_arxiv_dataset.ipynb), [CIFAR-10](notebooks/dataset_setup/prepare_cifar10_dataset.ipynb), and [Tabula Sapiens](notebooks/dataset_setup/prepare_tabula_sapiens_dataset.ipynb) setup notebooks.
536
-
537
- Earlier exploratory notebooks are in
538
- [notebooks/archive](notebooks/archive/). This includes the [interactive 2D
539
- embedding](notebooks/archive/digits_2d_interactive.ipynb), [interactive 3D
540
- embedding](notebooks/archive/digits_3d_interactive.ipynb), [parameter
541
- sweep](notebooks/archive/digits_parameter_sweep.ipynb), and [MST scaling
542
- benchmark](notebooks/archive/mst_scaling_benchmark.ipynb).
535
+ Prepare source data with the [arXiv](notebooks/dataset_setup/prepare_arxiv_dataset.ipynb), [CIFAR-100](notebooks/dataset_setup/prepare_cifar100_dataset.ipynb), and [Tabula Sapiens](notebooks/dataset_setup/prepare_tabula_sapiens_dataset.ipynb) setup notebooks.
543
536
 
544
537
  Python helper modules used by the notebooks are in [notebooks/src](notebooks/src/).
545
538
  For the full method description and equations, see the
@@ -1,5 +1,7 @@
1
1
  # IMSTE
2
2
 
3
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.24307529.svg)](https://doi.org/10.5281/zenodo.24307529)
4
+
3
5
  **Iterated Minimum Spanning Tree Embedding (IMSTE)** is a scikit-learn
4
6
  transformer for learning low-dimensional coordinates from numeric data. It
5
7
  builds a weighted graph from ranked minimum spanning trees, then
@@ -42,9 +44,6 @@ python -m pip install "imste[notebook]"
42
44
 
43
45
  From a source checkout, use `python -m pip install ".[notebook]"` instead.
44
46
 
45
- For maintainers preparing a release, see the
46
- [release guide](docs/releasing.md).
47
-
48
47
  Both estimators accept dense arrays and SciPy CSR matrices. The graph builders
49
48
  keep CSR inputs sparse. `InductiveIMSTE` densifies only each MLP mini-batch.
50
49
  The default FAMST graph builder uses RINND for dense Euclidean inputs and
@@ -309,7 +308,7 @@ directly; they use `embedding_` or the inductive model's `reference_embedding_`.
309
308
  Raw coordinate arrays remain supported.
310
309
 
311
310
  For class-level graph overlays across one or more embeddings, use
312
- `imste.class_graph.plot_class_mixture_mst`. It fits full-covariance Gaussian
311
+ `imste.visualization.plot_class_mixture_mst`. It fits full-covariance Gaussian
313
312
  mixtures per class in each 2D embedding, connects their component centers with
314
313
  iterated MSTs, and maps symmetric cross-component posterior probability to
315
314
  edge opacity. For each edge, the two endpoint Gaussian densities form an
@@ -478,21 +477,14 @@ optimization by passing an `IteratedMinimumSpanningTreeEmbedder` as `embedder`.
478
477
  Active experiment notebooks use concise dataset-based names. Dataset setup notebooks live in `notebooks/dataset_setup/`.
479
478
 
480
479
  - [synthetic_exp](notebooks/synthetic_exp.ipynb): compare IMSTE and UMAP on controlled geometries.
481
- - [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories.
480
+ - [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories. Prepare its local inputs with [prepare_biological_datasets](notebooks/dataset_setup/prepare_biological_datasets.ipynb).
482
481
  - [arxiv_exp](notebooks/arxiv_exp.ipynb): compare embeddings of the prepared arXiv sample.
483
- - [cifar10_exp](notebooks/cifar10_exp.ipynb): compare CIFAR-10 image-vector embeddings.
482
+ - [cifar100_exp](notebooks/cifar100_exp.ipynb): compare CIFAR-100 image embeddings with superclass hues and fine-class shades.
484
483
  - [interactive](notebooks/interactive.ipynb): interactively refine a balanced arXiv layout with gravity followed by mean shift.
485
484
  - [mnist_exp](notebooks/mnist_exp.ipynb): compare MNIST, Fashion-MNIST, and Kuzushiji-MNIST.
486
485
  - [tabula_sapiens_exp](notebooks/tabula_sapiens_exp.ipynb): load prepared Tabula Sapiens features and compare tissue and cell-type colorings on shared layouts.
487
486
 
488
- Prepare source data with the [arXiv](notebooks/dataset_setup/prepare_arxiv_dataset.ipynb), [CIFAR-10](notebooks/dataset_setup/prepare_cifar10_dataset.ipynb), and [Tabula Sapiens](notebooks/dataset_setup/prepare_tabula_sapiens_dataset.ipynb) setup notebooks.
489
-
490
- Earlier exploratory notebooks are in
491
- [notebooks/archive](notebooks/archive/). This includes the [interactive 2D
492
- embedding](notebooks/archive/digits_2d_interactive.ipynb), [interactive 3D
493
- embedding](notebooks/archive/digits_3d_interactive.ipynb), [parameter
494
- sweep](notebooks/archive/digits_parameter_sweep.ipynb), and [MST scaling
495
- benchmark](notebooks/archive/mst_scaling_benchmark.ipynb).
487
+ Prepare source data with the [arXiv](notebooks/dataset_setup/prepare_arxiv_dataset.ipynb), [CIFAR-100](notebooks/dataset_setup/prepare_cifar100_dataset.ipynb), and [Tabula Sapiens](notebooks/dataset_setup/prepare_tabula_sapiens_dataset.ipynb) setup notebooks.
496
488
 
497
489
  Python helper modules used by the notebooks are in [notebooks/src](notebooks/src/).
498
490
  For the full method description and equations, see the
@@ -1,4 +1,4 @@
1
- """Reusable Gaussian-mixture MST graph plots for low-dimensional embeddings."""
1
+ """Reusable class-mixture plots for labeled low-dimensional embeddings."""
2
2
 
3
3
  from __future__ import annotations
4
4
 
@@ -10,114 +10,11 @@ import pandas as pd
10
10
  from matplotlib.colors import BoundaryNorm, ListedColormap
11
11
  from matplotlib.lines import Line2D
12
12
  from matplotlib.patches import FancyArrowPatch
13
- from scipy.special import expit
14
- from scipy.stats import multivariate_normal
15
- from sklearn.mixture import GaussianMixture
16
- from imste.mst.prim import iterated_prim_edges
17
13
 
18
- NODE_EDGE_WIDTH = 1.1
19
-
20
-
21
- def _fit_class_mixture_nodes(
22
- embedding: np.ndarray,
23
- labels: np.ndarray,
24
- *,
25
- max_components: int,
26
- min_component_size: int,
27
- random_state: int,
28
- ) -> tuple[list[dict[str, object]], np.ndarray, np.ndarray]:
29
- """Fit full-covariance GMMs per class and return component nodes and labels."""
30
- classes = np.unique(labels)
31
- nodes: list[dict[str, object]] = []
32
- sample_node_ids = np.full(len(labels), -1, dtype=np.intp)
33
- class_codes = np.empty(len(labels), dtype=np.intp)
34
-
35
- for class_code, class_label in enumerate(classes):
36
- sample_indices = np.flatnonzero(labels == class_label)
37
- class_codes[sample_indices] = class_code
38
- n_components = min(
39
- max_components,
40
- max(1, len(sample_indices) // min_component_size),
41
- )
42
- while True:
43
- mixture = GaussianMixture(
44
- n_components=n_components,
45
- covariance_type="full",
46
- reg_covar=1e-5,
47
- n_init=3,
48
- random_state=random_state + class_code,
49
- ).fit(embedding[sample_indices])
50
- component_ids = mixture.predict(embedding[sample_indices])
51
- counts = np.bincount(component_ids, minlength=n_components)
52
- if n_components == 1 or counts.min() >= min_component_size:
53
- break
54
- n_components -= 1
55
-
56
- for component_id in range(n_components):
57
- members = sample_indices[component_ids == component_id]
58
- node_id = len(nodes)
59
- sample_node_ids[members] = node_id
60
- nodes.append(
61
- {
62
- "class_label": class_label,
63
- "class_code": class_code,
64
- "component_id": component_id,
65
- "position": mixture.means_[component_id].copy(),
66
- "covariance": mixture.covariances_[component_id].copy(),
67
- "mixture_weight": float(mixture.weights_[component_id]),
68
- "sample_indices": members,
69
- "sample_coordinates": embedding[members].copy(),
70
- "n_samples": len(members),
71
- }
72
- )
73
-
74
- if np.any(sample_node_ids < 0):
75
- raise RuntimeError("Some samples were not assigned to a class-mixture component.")
76
- return nodes, sample_node_ids, class_codes
14
+ from .class_graph import _edge_component_overlap, _fit_class_mixture_nodes
15
+ from .mst.prim import iterated_prim_edges
77
16
 
78
-
79
- def _edge_component_overlap(
80
- nodes: list[dict[str, object]],
81
- left: int,
82
- right: int,
83
- ) -> tuple[float, float, float]:
84
- """Return symmetric cross-component posterior overlap in 2D."""
85
- node_a, node_b = nodes[left], nodes[right]
86
- if node_a["class_label"] == node_b["class_label"]:
87
- raise ValueError("Same-class edges should be filtered before scoring.")
88
- indices_a = np.asarray(node_a["sample_indices"], dtype=np.intp)
89
- indices_b = np.asarray(node_b["sample_indices"], dtype=np.intp)
90
- covariance_a = np.asarray(node_a["covariance"])
91
- covariance_b = np.asarray(node_b["covariance"])
92
- log_density_a_on_b = multivariate_normal.logpdf(
93
- np.asarray(node_b["sample_coordinates"]),
94
- mean=np.asarray(node_a["position"]),
95
- cov=covariance_a,
96
- allow_singular=True,
97
- )
98
- log_density_b_on_b = multivariate_normal.logpdf(
99
- np.asarray(node_b["sample_coordinates"]),
100
- mean=np.asarray(node_b["position"]),
101
- cov=covariance_b,
102
- allow_singular=True,
103
- )
104
- log_density_a_on_a = multivariate_normal.logpdf(
105
- np.asarray(node_a["sample_coordinates"]),
106
- mean=np.asarray(node_a["position"]),
107
- cov=covariance_a,
108
- allow_singular=True,
109
- )
110
- log_density_b_on_a = multivariate_normal.logpdf(
111
- np.asarray(node_a["sample_coordinates"]),
112
- mean=np.asarray(node_b["position"]),
113
- cov=covariance_b,
114
- allow_singular=True,
115
- )
116
- # Equal priors make this an endpoint distinguishability score, independent
117
- # of the relative number of samples in each component.
118
- prob_a_to_b = float(np.mean(expit(log_density_a_on_b - log_density_b_on_b)))
119
- prob_b_to_a = float(np.mean(expit(log_density_b_on_a - log_density_a_on_a)))
120
- return prob_a_to_b, prob_b_to_a, (prob_a_to_b + prob_b_to_a) / 2.0
17
+ NODE_EDGE_WIDTH = 1.1
121
18
 
122
19
 
123
20
  def _overlap_to_alpha(score: float, faint: float, opaque: float) -> float:
@@ -126,103 +23,6 @@ def _overlap_to_alpha(score: float, faint: float, opaque: float) -> float:
126
23
  return float(faint + confusion * (opaque - faint))
127
24
 
128
25
 
129
- def compare_class_graph_relations(
130
- edge_scores: pd.DataFrame,
131
- *,
132
- method_a: str = "IMSTE",
133
- method_b: str = "UMAP",
134
- dataset: object | None = None,
135
- ) -> pd.DataFrame:
136
- """Rank class-pair MST relations unique to either embedding.
137
-
138
- Component-level edges are collapsed to unordered class pairs because the
139
- GMM components are fit independently in each embedding. A relation is
140
- considered present when at least one cross-class component edge connects
141
- that class pair. Duplicate appearances across iterated MSTs count once.
142
- ``edge_error`` is the mean chance-normalized symmetric cross-assignment
143
- probability across the distinct component edges for that class pair; the
144
- table also reports the maximum edge error and component-level details.
145
-
146
- Rows are sorted by descending edge error separately for relations unique
147
- to ``method_a`` and ``method_b``. A chance-level cross-assignment
148
- probability of 0.5 maps to edge error 1.0; perfect separation maps to 0.
149
- """
150
- required = {
151
- "embedding", "class_a", "component_a", "class_b", "component_b",
152
- "symmetric_component_overlap",
153
- }
154
- missing = required.difference(edge_scores.columns)
155
- if missing:
156
- raise ValueError(f"edge_scores is missing columns: {sorted(missing)}")
157
- if method_a == method_b:
158
- raise ValueError("method_a and method_b must be different embedding names.")
159
- frame = edge_scores.copy()
160
- if dataset is not None:
161
- if "dataset" not in frame:
162
- raise ValueError("dataset was specified but edge_scores has no dataset column.")
163
- frame = frame.loc[frame["dataset"] == dataset]
164
- frame = frame.loc[frame["embedding"].isin([method_a, method_b])]
165
-
166
- def label_key(value: object) -> tuple[str, str]:
167
- return type(value).__name__, str(value)
168
-
169
- # method -> unordered label pair -> component edge -> edge error
170
- relations: dict[str, dict[tuple[object, object], dict[tuple[object, object], float]]] = {
171
- method_a: {}, method_b: {}
172
- }
173
- for row in frame.to_dict("records"):
174
- left, right = row["class_a"], row["class_b"]
175
- if label_key(left) <= label_key(right):
176
- class_pair = (left, right)
177
- component_pair = (row["component_a"], row["component_b"])
178
- else:
179
- class_pair = (right, left)
180
- component_pair = (row["component_b"], row["component_a"])
181
- error = float(np.clip(row["symmetric_component_overlap"] / 0.5, 0.0, 1.0))
182
- method = row["embedding"]
183
- relation_edges = relations[method].setdefault(class_pair, {})
184
- # The same component edge can appear in multiple iterated MSTs.
185
- relation_edges[component_pair] = max(error, relation_edges.get(component_pair, 0.0))
186
-
187
- rows: list[dict[str, object]] = []
188
- for source, target in ((method_a, method_b), (method_b, method_a)):
189
- unique_pairs = set(relations[source]).difference(relations[target])
190
- ranked = []
191
- for class_pair in unique_pairs:
192
- component_errors = relations[source][class_pair]
193
- errors = list(component_errors.values())
194
- ranked.append(
195
- {
196
- "class_a": class_pair[0],
197
- "class_b": class_pair[1],
198
- "present_in": source,
199
- "missing_from": target,
200
- "edge_error": float(np.mean(errors)),
201
- "max_component_edge_error": float(np.max(errors)),
202
- "n_component_edges": len(errors),
203
- "component_edges": tuple(
204
- (component_a, component_b, component_errors[(component_a, component_b)])
205
- for component_a, component_b in sorted(
206
- component_errors, key=lambda pair: (str(pair[0]), str(pair[1]))
207
- )
208
- ),
209
- }
210
- )
211
- ranked.sort(key=lambda row: (-row["edge_error"], -row["max_component_edge_error"],
212
- label_key(row["class_a"]), label_key(row["class_b"])))
213
- for rank, row in enumerate(ranked, start=1):
214
- rows.append({"rank": rank, **row})
215
-
216
- return pd.DataFrame(
217
- rows,
218
- columns=[
219
- "rank", "class_a", "class_b", "present_in", "missing_from",
220
- "edge_error", "max_component_edge_error", "n_component_edges",
221
- "component_edges",
222
- ],
223
- )
224
-
225
-
226
26
  def plot_class_mixture_mst(
227
27
  X: np.ndarray,
228
28
  labels: object,
@@ -0,0 +1,243 @@
1
+ """Analysis helpers for class-mixture minimum-spanning-tree relations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping
6
+
7
+ import numpy as np
8
+ import pandas as pd
9
+ from scipy.special import expit
10
+ from scipy.stats import multivariate_normal
11
+ from sklearn.mixture import GaussianMixture
12
+
13
+ def _fit_class_mixture_nodes(
14
+ embedding: np.ndarray,
15
+ labels: np.ndarray,
16
+ *,
17
+ max_components: int,
18
+ min_component_size: int,
19
+ random_state: int,
20
+ ) -> tuple[list[dict[str, object]], np.ndarray, np.ndarray]:
21
+ """Fit full-covariance GMMs per class and return component nodes and labels."""
22
+ classes = np.unique(labels)
23
+ nodes: list[dict[str, object]] = []
24
+ sample_node_ids = np.full(len(labels), -1, dtype=np.intp)
25
+ class_codes = np.empty(len(labels), dtype=np.intp)
26
+
27
+ for class_code, class_label in enumerate(classes):
28
+ sample_indices = np.flatnonzero(labels == class_label)
29
+ class_codes[sample_indices] = class_code
30
+ n_components = min(
31
+ max_components,
32
+ max(1, len(sample_indices) // min_component_size),
33
+ )
34
+ while True:
35
+ mixture = GaussianMixture(
36
+ n_components=n_components,
37
+ covariance_type="full",
38
+ reg_covar=1e-5,
39
+ n_init=3,
40
+ random_state=random_state + class_code,
41
+ ).fit(embedding[sample_indices])
42
+ component_ids = mixture.predict(embedding[sample_indices])
43
+ counts = np.bincount(component_ids, minlength=n_components)
44
+ if n_components == 1 or counts.min() >= min_component_size:
45
+ break
46
+ n_components -= 1
47
+
48
+ for component_id in range(n_components):
49
+ members = sample_indices[component_ids == component_id]
50
+ node_id = len(nodes)
51
+ sample_node_ids[members] = node_id
52
+ nodes.append(
53
+ {
54
+ "class_label": class_label,
55
+ "class_code": class_code,
56
+ "component_id": component_id,
57
+ "position": mixture.means_[component_id].copy(),
58
+ "covariance": mixture.covariances_[component_id].copy(),
59
+ "mixture_weight": float(mixture.weights_[component_id]),
60
+ "sample_indices": members,
61
+ "sample_coordinates": embedding[members].copy(),
62
+ "n_samples": len(members),
63
+ }
64
+ )
65
+
66
+ if np.any(sample_node_ids < 0):
67
+ raise RuntimeError("Some samples were not assigned to a class-mixture component.")
68
+ return nodes, sample_node_ids, class_codes
69
+
70
+
71
+ def _edge_component_overlap(
72
+ nodes: list[dict[str, object]],
73
+ left: int,
74
+ right: int,
75
+ ) -> tuple[float, float, float]:
76
+ """Return symmetric cross-component posterior overlap in 2D."""
77
+ node_a, node_b = nodes[left], nodes[right]
78
+ if node_a["class_label"] == node_b["class_label"]:
79
+ raise ValueError("Same-class edges should be filtered before scoring.")
80
+ indices_a = np.asarray(node_a["sample_indices"], dtype=np.intp)
81
+ indices_b = np.asarray(node_b["sample_indices"], dtype=np.intp)
82
+ covariance_a = np.asarray(node_a["covariance"])
83
+ covariance_b = np.asarray(node_b["covariance"])
84
+ log_density_a_on_b = multivariate_normal.logpdf(
85
+ np.asarray(node_b["sample_coordinates"]),
86
+ mean=np.asarray(node_a["position"]),
87
+ cov=covariance_a,
88
+ allow_singular=True,
89
+ )
90
+ log_density_b_on_b = multivariate_normal.logpdf(
91
+ np.asarray(node_b["sample_coordinates"]),
92
+ mean=np.asarray(node_b["position"]),
93
+ cov=covariance_b,
94
+ allow_singular=True,
95
+ )
96
+ log_density_a_on_a = multivariate_normal.logpdf(
97
+ np.asarray(node_a["sample_coordinates"]),
98
+ mean=np.asarray(node_a["position"]),
99
+ cov=covariance_a,
100
+ allow_singular=True,
101
+ )
102
+ log_density_b_on_a = multivariate_normal.logpdf(
103
+ np.asarray(node_a["sample_coordinates"]),
104
+ mean=np.asarray(node_b["position"]),
105
+ cov=covariance_b,
106
+ allow_singular=True,
107
+ )
108
+ # Equal priors make this an endpoint distinguishability score, independent
109
+ # of the relative number of samples in each component.
110
+ prob_a_to_b = float(np.mean(expit(log_density_a_on_b - log_density_b_on_b)))
111
+ prob_b_to_a = float(np.mean(expit(log_density_b_on_a - log_density_a_on_a)))
112
+ return prob_a_to_b, prob_b_to_a, (prob_a_to_b + prob_b_to_a) / 2.0
113
+
114
+
115
+ def compare_class_graph_relations(
116
+ edge_scores: pd.DataFrame,
117
+ *,
118
+ method_a: str = "IMSTE",
119
+ method_b: str = "UMAP",
120
+ dataset: object | None = None,
121
+ ) -> pd.DataFrame:
122
+ """Rank class-pair MST relations unique to either embedding.
123
+
124
+ Component-level edges are collapsed to unordered class pairs because the
125
+ GMM components are fit independently in each embedding. A relation is
126
+ considered present when at least one cross-class component edge connects
127
+ that class pair. Duplicate appearances across iterated MSTs count once.
128
+ ``edge_error`` is the mean chance-normalized symmetric cross-assignment
129
+ probability across the distinct component edges for that class pair; the
130
+ table also reports the maximum edge error and component-level details.
131
+
132
+ Rows are sorted by descending edge error separately for relations unique
133
+ to ``method_a`` and ``method_b``. A chance-level cross-assignment
134
+ probability of 0.5 maps to edge error 1.0; perfect separation maps to 0.
135
+ """
136
+ required = {
137
+ "embedding", "class_a", "component_a", "class_b", "component_b",
138
+ "symmetric_component_overlap",
139
+ }
140
+ missing = required.difference(edge_scores.columns)
141
+ if missing:
142
+ raise ValueError(f"edge_scores is missing columns: {sorted(missing)}")
143
+ if method_a == method_b:
144
+ raise ValueError("method_a and method_b must be different embedding names.")
145
+ frame = edge_scores.copy()
146
+ if dataset is not None:
147
+ if "dataset" not in frame:
148
+ raise ValueError("dataset was specified but edge_scores has no dataset column.")
149
+ frame = frame.loc[frame["dataset"] == dataset]
150
+ frame = frame.loc[frame["embedding"].isin([method_a, method_b])]
151
+
152
+ def label_key(value: object) -> tuple[str, str]:
153
+ return type(value).__name__, str(value)
154
+
155
+ # method -> unordered label pair -> component edge -> edge error
156
+ relations: dict[str, dict[tuple[object, object], dict[tuple[object, object], float]]] = {
157
+ method_a: {}, method_b: {}
158
+ }
159
+ for row in frame.to_dict("records"):
160
+ left, right = row["class_a"], row["class_b"]
161
+ if label_key(left) <= label_key(right):
162
+ class_pair = (left, right)
163
+ component_pair = (row["component_a"], row["component_b"])
164
+ else:
165
+ class_pair = (right, left)
166
+ component_pair = (row["component_b"], row["component_a"])
167
+ error = float(np.clip(row["symmetric_component_overlap"] / 0.5, 0.0, 1.0))
168
+ method = row["embedding"]
169
+ relation_edges = relations[method].setdefault(class_pair, {})
170
+ # The same component edge can appear in multiple iterated MSTs.
171
+ relation_edges[component_pair] = max(error, relation_edges.get(component_pair, 0.0))
172
+
173
+ rows: list[dict[str, object]] = []
174
+ for source, target in ((method_a, method_b), (method_b, method_a)):
175
+ unique_pairs = set(relations[source]).difference(relations[target])
176
+ ranked = []
177
+ for class_pair in unique_pairs:
178
+ component_errors = relations[source][class_pair]
179
+ errors = list(component_errors.values())
180
+ ranked.append(
181
+ {
182
+ "class_a": class_pair[0],
183
+ "class_b": class_pair[1],
184
+ "present_in": source,
185
+ "missing_from": target,
186
+ "edge_error": float(np.mean(errors)),
187
+ "max_component_edge_error": float(np.max(errors)),
188
+ "n_component_edges": len(errors),
189
+ "component_edges": tuple(
190
+ (component_a, component_b, component_errors[(component_a, component_b)])
191
+ for component_a, component_b in sorted(
192
+ component_errors, key=lambda pair: (str(pair[0]), str(pair[1]))
193
+ )
194
+ ),
195
+ }
196
+ )
197
+ ranked.sort(key=lambda row: (-row["edge_error"], -row["max_component_edge_error"],
198
+ label_key(row["class_a"]), label_key(row["class_b"])))
199
+ for rank, row in enumerate(ranked, start=1):
200
+ rows.append({"rank": rank, **row})
201
+
202
+ return pd.DataFrame(
203
+ rows,
204
+ columns=[
205
+ "rank", "class_a", "class_b", "present_in", "missing_from",
206
+ "edge_error", "max_component_edge_error", "n_component_edges",
207
+ "component_edges",
208
+ ],
209
+ )
210
+
211
+
212
+ def plot_class_mixture_mst(
213
+ X: np.ndarray,
214
+ labels: object,
215
+ embeddings: Mapping[str, object],
216
+ *,
217
+ title: str | None = None,
218
+ n_msts: int = 2,
219
+ max_components_per_class: int = 3,
220
+ min_component_size: int = 20,
221
+ point_alpha: float = 0.4,
222
+ min_edge_alpha: float = 0.2,
223
+ max_edge_alpha: float = 0.9,
224
+ random_state: int = 42,
225
+ class_colors: Mapping[object, object] | None = None,
226
+ ):
227
+ """Backward-compatible import for the plotting API in ``imste.visualization``."""
228
+ from .visualization import plot_class_mixture_mst as plotter
229
+
230
+ return plotter(
231
+ X,
232
+ labels,
233
+ embeddings,
234
+ title=title,
235
+ n_msts=n_msts,
236
+ max_components_per_class=max_components_per_class,
237
+ min_component_size=min_component_size,
238
+ point_alpha=point_alpha,
239
+ min_edge_alpha=min_edge_alpha,
240
+ max_edge_alpha=max_edge_alpha,
241
+ random_state=random_state,
242
+ class_colors=class_colors,
243
+ )
@@ -19,10 +19,12 @@ from ._visualization_class_geometry import (
19
19
  plot_ellipses,
20
20
  plot_knn_union,
21
21
  )
22
+ from ._visualization_class_graph import plot_class_mixture_mst
22
23
 
23
24
  __all__ = [
24
25
  "cross_validate_embedding_knn",
25
26
  "interactive_force",
27
+ "plot_class_mixture_mst",
26
28
  "plot_density_parent_tree",
27
29
  "plot_ellipses",
28
30
  "plot_embedding",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: imste
3
- Version: 0.2.2
3
+ Version: 0.3.0
4
4
  Summary: Iterated Minimum Spanning Tree Embedding (IMSTE)
5
5
  Author: Fabrizio Costa
6
6
  License-Expression: MIT
@@ -33,6 +33,7 @@ Provides-Extra: test
33
33
  Requires-Dist: pytest>=7; extra == "test"
34
34
  Requires-Dist: pandas>=2.0; extra == "test"
35
35
  Requires-Dist: torch>=2.0; extra == "test"
36
+ Requires-Dist: matplotlib>=3.7; extra == "test"
36
37
  Provides-Extra: notebook
37
38
  Requires-Dist: matplotlib>=3.7; extra == "notebook"
38
39
  Requires-Dist: pandas>=2.0; extra == "notebook"
@@ -47,6 +48,8 @@ Dynamic: license-file
47
48
 
48
49
  # IMSTE
49
50
 
51
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.24307529.svg)](https://doi.org/10.5281/zenodo.24307529)
52
+
50
53
  **Iterated Minimum Spanning Tree Embedding (IMSTE)** is a scikit-learn
51
54
  transformer for learning low-dimensional coordinates from numeric data. It
52
55
  builds a weighted graph from ranked minimum spanning trees, then
@@ -89,9 +92,6 @@ python -m pip install "imste[notebook]"
89
92
 
90
93
  From a source checkout, use `python -m pip install ".[notebook]"` instead.
91
94
 
92
- For maintainers preparing a release, see the
93
- [release guide](docs/releasing.md).
94
-
95
95
  Both estimators accept dense arrays and SciPy CSR matrices. The graph builders
96
96
  keep CSR inputs sparse. `InductiveIMSTE` densifies only each MLP mini-batch.
97
97
  The default FAMST graph builder uses RINND for dense Euclidean inputs and
@@ -356,7 +356,7 @@ directly; they use `embedding_` or the inductive model's `reference_embedding_`.
356
356
  Raw coordinate arrays remain supported.
357
357
 
358
358
  For class-level graph overlays across one or more embeddings, use
359
- `imste.class_graph.plot_class_mixture_mst`. It fits full-covariance Gaussian
359
+ `imste.visualization.plot_class_mixture_mst`. It fits full-covariance Gaussian
360
360
  mixtures per class in each 2D embedding, connects their component centers with
361
361
  iterated MSTs, and maps symmetric cross-component posterior probability to
362
362
  edge opacity. For each edge, the two endpoint Gaussian densities form an
@@ -525,21 +525,14 @@ optimization by passing an `IteratedMinimumSpanningTreeEmbedder` as `embedder`.
525
525
  Active experiment notebooks use concise dataset-based names. Dataset setup notebooks live in `notebooks/dataset_setup/`.
526
526
 
527
527
  - [synthetic_exp](notebooks/synthetic_exp.ipynb): compare IMSTE and UMAP on controlled geometries.
528
- - [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories.
528
+ - [biological_exp](notebooks/biological_exp.ipynb): inspect annotated biological datasets and trajectories. Prepare its local inputs with [prepare_biological_datasets](notebooks/dataset_setup/prepare_biological_datasets.ipynb).
529
529
  - [arxiv_exp](notebooks/arxiv_exp.ipynb): compare embeddings of the prepared arXiv sample.
530
- - [cifar10_exp](notebooks/cifar10_exp.ipynb): compare CIFAR-10 image-vector embeddings.
530
+ - [cifar100_exp](notebooks/cifar100_exp.ipynb): compare CIFAR-100 image embeddings with superclass hues and fine-class shades.
531
531
  - [interactive](notebooks/interactive.ipynb): interactively refine a balanced arXiv layout with gravity followed by mean shift.
532
532
  - [mnist_exp](notebooks/mnist_exp.ipynb): compare MNIST, Fashion-MNIST, and Kuzushiji-MNIST.
533
533
  - [tabula_sapiens_exp](notebooks/tabula_sapiens_exp.ipynb): load prepared Tabula Sapiens features and compare tissue and cell-type colorings on shared layouts.
534
534
 
535
- Prepare source data with the [arXiv](notebooks/dataset_setup/prepare_arxiv_dataset.ipynb), [CIFAR-10](notebooks/dataset_setup/prepare_cifar10_dataset.ipynb), and [Tabula Sapiens](notebooks/dataset_setup/prepare_tabula_sapiens_dataset.ipynb) setup notebooks.
536
-
537
- Earlier exploratory notebooks are in
538
- [notebooks/archive](notebooks/archive/). This includes the [interactive 2D
539
- embedding](notebooks/archive/digits_2d_interactive.ipynb), [interactive 3D
540
- embedding](notebooks/archive/digits_3d_interactive.ipynb), [parameter
541
- sweep](notebooks/archive/digits_parameter_sweep.ipynb), and [MST scaling
542
- benchmark](notebooks/archive/mst_scaling_benchmark.ipynb).
535
+ Prepare source data with the [arXiv](notebooks/dataset_setup/prepare_arxiv_dataset.ipynb), [CIFAR-100](notebooks/dataset_setup/prepare_cifar100_dataset.ipynb), and [Tabula Sapiens](notebooks/dataset_setup/prepare_tabula_sapiens_dataset.ipynb) setup notebooks.
543
536
 
544
537
  Python helper modules used by the notebooks are in [notebooks/src](notebooks/src/).
545
538
  For the full method description and equations, see the
@@ -14,6 +14,7 @@ imste/_optimizer_mps.py
14
14
  imste/_optimizer_torch.py
15
15
  imste/_progress.py
16
16
  imste/_visualization_class_geometry.py
17
+ imste/_visualization_class_graph.py
17
18
  imste/_visualization_core.py
18
19
  imste/_visualization_interactive.py
19
20
  imste/class_graph.py
@@ -33,4 +34,5 @@ tests/test_arxiv_notebook_helpers.py
33
34
  tests/test_estimator.py
34
35
  tests/test_famst.py
35
36
  tests/test_multilevel.py
36
- tests/test_optimizer.py
37
+ tests/test_optimizer.py
38
+ tests/test_visualization.py
@@ -26,3 +26,4 @@ yt>=4.4
26
26
  pytest>=7
27
27
  pandas>=2.0
28
28
  torch>=2.0
29
+ matplotlib>=3.7
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "imste"
7
- version = "0.2.2"
7
+ version = "0.3.0"
8
8
  description = "Iterated Minimum Spanning Tree Embedding (IMSTE)"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -40,7 +40,7 @@ Documentation = "https://github.com/fabriziocosta/IteratedMinimumSpanningTree/tr
40
40
 
41
41
  [project.optional-dependencies]
42
42
  gpu = ["torch>=2.0"]
43
- test = ["pytest>=7", "pandas>=2.0", "torch>=2.0"]
43
+ test = ["pytest>=7", "pandas>=2.0", "torch>=2.0", "matplotlib>=3.7"]
44
44
  notebook = [
45
45
  "matplotlib>=3.7",
46
46
  "pandas>=2.0",
@@ -0,0 +1,68 @@
1
+ import matplotlib
2
+ import numpy as np
3
+ import pandas as pd
4
+ import pytest
5
+
6
+ matplotlib.use("Agg", force=True)
7
+ import matplotlib.pyplot as plt
8
+
9
+ from imste.class_graph import plot_class_mixture_mst as legacy_plot_class_mixture_mst
10
+ from imste.visualization import plot_class_mixture_mst, plot_embedding
11
+
12
+
13
+ def test_plot_embedding_accepts_series_and_returns_figure_and_axes():
14
+ coordinates = np.array([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]])
15
+ labels = pd.Series(["a", "a", "b", "b"], name="class")
16
+
17
+ figure, axes = plot_embedding(coordinates, labels)
18
+
19
+ assert figure is axes[0, 0].figure
20
+ assert axes.shape == (1, 1)
21
+ assert axes[0, 0].get_title() == "IMSTE colored by class"
22
+ plt.close(figure)
23
+
24
+
25
+ def test_plot_embedding_rejects_label_count_mismatch():
26
+ with pytest.raises(ValueError, match="obs rows must match"):
27
+ plot_embedding(np.zeros((3, 2)), ["a", "b"])
28
+
29
+
30
+ def test_class_mixture_plot_is_in_visualization_api_and_legacy_import_still_works(
31
+ monkeypatch,
32
+ ):
33
+ expected = object()
34
+ monkeypatch.setattr("imste.visualization.plot_class_mixture_mst", lambda *a, **k: expected)
35
+
36
+ result = legacy_plot_class_mixture_mst(
37
+ np.zeros((2, 1)), np.array(["a", "b"]), {"IMSTE": np.zeros((2, 2))}
38
+ )
39
+
40
+ assert result is expected
41
+ assert callable(plot_class_mixture_mst)
42
+
43
+
44
+ def test_class_mixture_plot_returns_axes_and_edge_scores():
45
+ rng = np.random.default_rng(2)
46
+ labels = np.repeat(["left", "right"], 24)
47
+ embedding = np.vstack(
48
+ [rng.normal(loc=(-2.0, 0.0), scale=0.25, size=(24, 2)),
49
+ rng.normal(loc=(2.0, 0.0), scale=0.25, size=(24, 2))]
50
+ )
51
+
52
+ figure, axes, edge_scores = plot_class_mixture_mst(
53
+ embedding,
54
+ labels,
55
+ {"IMSTE": embedding, "UMAP": embedding + np.array([0.1, -0.1])},
56
+ n_msts=1,
57
+ max_components_per_class=1,
58
+ min_component_size=5,
59
+ random_state=0,
60
+ )
61
+
62
+ assert len(axes) == 2
63
+ assert {"embedding", "class_a", "class_b", "symmetric_component_overlap"}.issubset(
64
+ edge_scores.columns
65
+ )
66
+ assert set(edge_scores["embedding"]) == {"IMSTE", "UMAP"}
67
+ assert len(edge_scores) == 2
68
+ plt.close(figure)
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes