threadfin 4.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- threadfin/__init__.py +86 -0
- threadfin/_utils.py +260 -0
- threadfin/api.py +658 -0
- threadfin/bcrgraph.py +255 -0
- threadfin/clones.py +653 -0
- threadfin/core.py +531 -0
- threadfin/dandelion.py +458 -0
- threadfin/density.py +409 -0
- threadfin/dynamics.py +275 -0
- threadfin/export.py +215 -0
- threadfin/expression.py +121 -0
- threadfin/heritability.py +224 -0
- threadfin/integrate.py +350 -0
- threadfin/io.py +280 -0
- threadfin/maptune.py +743 -0
- threadfin/metrics.py +137 -0
- threadfin/migration.py +165 -0
- threadfin/phylo.py +834 -0
- threadfin/plotting.py +535 -0
- threadfin/pp.py +253 -0
- threadfin/profiles.py +603 -0
- threadfin/programmes.py +1028 -0
- threadfin/programs.py +133 -0
- threadfin/reclustering.py +68 -0
- threadfin/sequence.py +379 -0
- threadfin/simulate.py +206 -0
- threadfin/specificity.py +355 -0
- threadfin/stats.py +476 -0
- threadfin/tl.py +62 -0
- threadfin-4.5.0.dist-info/METADATA +288 -0
- threadfin-4.5.0.dist-info/RECORD +34 -0
- threadfin-4.5.0.dist-info/WHEEL +5 -0
- threadfin-4.5.0.dist-info/licenses/LICENSE +21 -0
- threadfin-4.5.0.dist-info/top_level.txt +1 -0
threadfin/maptune.py
ADDED
|
@@ -0,0 +1,743 @@
|
|
|
1
|
+
"""Automatic choice of the clone-map layout.
|
|
2
|
+
|
|
3
|
+
The clone map (:func:`threadfin.tl.embed_clones`) is a UMAP of the clone profiles. Two settings decide
|
|
4
|
+
what it looks like: ``n_neighbors``, the number of profile neighbours each clone is tied to, and
|
|
5
|
+
``min_dist``, how closely clones may pack on the map. Too few neighbours or too little spacing breaks a
|
|
6
|
+
continuum of clones into separate islands; too many smooths away local structure. The best values differ
|
|
7
|
+
between datasets, because the number of clones and the shape of their distribution differ.
|
|
8
|
+
:func:`tune_clone_map` chooses both settings from the data, so that the map is as faithful to the clone
|
|
9
|
+
profiles as possible while holding together as one picture.
|
|
10
|
+
|
|
11
|
+
What is scored
|
|
12
|
+
--------------
|
|
13
|
+
Each layout is scored on three properties that need no annotation (:func:`map_quality`):
|
|
14
|
+
|
|
15
|
+
* **coherence**: the fraction of clones in the largest connected piece of the map. Two clones are
|
|
16
|
+
connected when they lie within ``1.5 D / sqrt(N)`` of each other (``D``, the extent of the map; ``N``,
|
|
17
|
+
the number of clones). At Scanpy's default point size (120,000 / N), this is roughly the distance at
|
|
18
|
+
which two drawn points touch, so a piece is what a reader sees as one island.
|
|
19
|
+
* **local fidelity**: the fraction of each clone's 30 nearest neighbours in profile space that are also
|
|
20
|
+
among its 30 nearest neighbours on the map (kNN preservation; Kobak & Berens 2019).
|
|
21
|
+
* **global fidelity**: the Spearman correlation between profile-space and map distances over random
|
|
22
|
+
pairs of clones (Kobak & Berens 2019).
|
|
23
|
+
|
|
24
|
+
The score is the mean of the two fidelities, minus a penalty when coherence falls below a target::
|
|
25
|
+
|
|
26
|
+
score = (local + global) / 2 - penalty * max(0, target - coherence)
|
|
27
|
+
|
|
28
|
+
By default the target is set by the clone profiles themselves: the fraction of clones in the largest
|
|
29
|
+
connected component of the mutual 15-nearest-neighbour graph of profiles (the graph scale used for
|
|
30
|
+
programme discovery), capped at 0.99. A map is therefore not allowed to be more fragmented than the
|
|
31
|
+
profiles it shows, while a small group of clones that is genuinely separate in profile space may stay
|
|
32
|
+
apart.
|
|
33
|
+
|
|
34
|
+
How the settings are searched
|
|
35
|
+
-----------------------------
|
|
36
|
+
A UMAP layout is random. The same settings can give a connected map with one seed and a broken one with
|
|
37
|
+
another, and the settings that are most faithful on one layout are often the ones closest to breaking
|
|
38
|
+
apart. The search is therefore built around the worst case rather than the typical layout.
|
|
39
|
+
|
|
40
|
+
1. **Ladder.** The clones are laid out at every combination of a ladder of neighbour counts (10, 18, 32,
|
|
41
|
+
56, 100 and 180, about 1.8-fold apart and at most half the number of clones) and of ``min_dist``
|
|
42
|
+
(0.3, 0.55, 0.8 and 0.95), each with two random seeds shared by all settings. A setting is credited
|
|
43
|
+
with its mean fidelity and with the *lowest* coherence among its layouts.
|
|
44
|
+
2. **Smoothing.** Each setting's score is averaged with those of its neighbours on the ladder (Gaussian
|
|
45
|
+
weights), so that a setting surrounded by good settings is preferred to an isolated peak at the edge
|
|
46
|
+
where the map starts to break apart.
|
|
47
|
+
3. **Confirmation.** The best settings are laid out with four further seeds and re-scored on all six
|
|
48
|
+
layouts.
|
|
49
|
+
4. **Choice.** Among the confirmed settings that held together in every layout, the one with the best
|
|
50
|
+
smoothed score wins, recomputed with all layouts so far, so that a setting whose neighbours on the
|
|
51
|
+
ladder break apart is not chosen even if it held together itself. Settings within ``tolerance`` of
|
|
52
|
+
the best are treated as equally good, and the most coherent of these is chosen.
|
|
53
|
+
5. **Rescue.** If the winner still falls short of the target in one of its layouts, the most coherent
|
|
54
|
+
settings seen so far are confirmed as well. If none of them holds together either, the most coherent
|
|
55
|
+
settings are laid out again with weaker repulsion between clones (UMAP ``repulsion_strength`` 0.5, then
|
|
56
|
+
0.25). Weaker repulsion moves UMAP along the attraction-repulsion spectrum towards more continuous
|
|
57
|
+
layouts (Böhm, Berens & Kobak 2022), which keeps weakly connected groups of clones together.
|
|
58
|
+
|
|
59
|
+
Layouts during the search use 200 optimisation epochs, which scores like the full layout at a fraction of
|
|
60
|
+
the cost; the final map is laid out at UMAP's defaults. Above ``max_clones`` clones, the search runs on a
|
|
61
|
+
random subset, ``n_neighbors`` is scaled up to the full data in proportion to the number of clones (which
|
|
62
|
+
keeps the neighbourhood the same size in profile space), and the best settings are confirmed again on all
|
|
63
|
+
clones before one is chosen.
|
|
64
|
+
|
|
65
|
+
``method="adam"`` replaces steps 1 and 2 with a stochastic-gradient search: Adam (Kingma & Ba 2015) on
|
|
66
|
+
``log n_neighbors`` and ``logit min_dist``, with gradients estimated by simultaneous perturbation (SPSA;
|
|
67
|
+
Spall 1992) from pairs of layouts that share a seed. It then goes through the same confirmation, choice
|
|
68
|
+
and rescue. The ladder is the default because it explores the whole range of settings and its choice
|
|
69
|
+
varies less between random seeds.
|
|
70
|
+
|
|
71
|
+
References
|
|
72
|
+
----------
|
|
73
|
+
Böhm, J. N., Berens, P. & Kobak, D. Attraction-repulsion spectrum in neighbor embeddings. J. Mach. Learn.
|
|
74
|
+
Res. 23, 1-32 (2022).
|
|
75
|
+
Kingma, D. P. & Ba, J. Adam: a method for stochastic optimization. ICLR (2015).
|
|
76
|
+
Kobak, D. & Berens, P. The art of using t-SNE for single-cell transcriptomics. Nat. Commun. 10, 5416 (2019).
|
|
77
|
+
Spall, J. C. Multivariate stochastic approximation using a simultaneous perturbation gradient
|
|
78
|
+
approximation. IEEE Trans. Autom. Control 37, 332-341 (1992).
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
from __future__ import annotations
|
|
82
|
+
|
|
83
|
+
import time
|
|
84
|
+
import warnings
|
|
85
|
+
from dataclasses import dataclass, field
|
|
86
|
+
from numbers import Real
|
|
87
|
+
|
|
88
|
+
import numpy as np
|
|
89
|
+
import pandas as pd
|
|
90
|
+
|
|
91
|
+
from ._utils import log
|
|
92
|
+
|
|
93
|
+
#: ladder of settings searched by the default method
|
|
94
|
+
N_NEIGHBORS_LADDER = (10, 18, 32, 56, 100, 180)
|
|
95
|
+
MIN_DIST_LADDER = (0.3, 0.55, 0.8, 0.95)
|
|
96
|
+
#: weaker repulsion tried, in this order, when no standard setting holds together
|
|
97
|
+
REPULSION_RESCUE = (0.5, 0.25)
|
|
98
|
+
#: settings never leave this box
|
|
99
|
+
N_NEIGHBORS_RANGE = (5, 250)
|
|
100
|
+
MIN_DIST_RANGE = (0.0, 0.99)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# --------------------------------------------------------------------------- scoring a layout
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _extent(xy: np.ndarray) -> float:
|
|
107
|
+
"""Side of the map: the larger of the two axis ranges, ignoring the outer 0.5% of clones."""
|
|
108
|
+
lo, hi = np.quantile(xy, [0.005, 0.995], axis=0)
|
|
109
|
+
return float(np.max(hi - lo))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _coherence(xy: np.ndarray, touch: float = 1.5) -> float:
|
|
113
|
+
"""Fraction of clones in the largest piece of the map (single linkage at ``touch * D / sqrt(N)``)."""
|
|
114
|
+
from scipy.sparse.csgraph import connected_components
|
|
115
|
+
from sklearn.neighbors import radius_neighbors_graph
|
|
116
|
+
|
|
117
|
+
n = len(xy)
|
|
118
|
+
radius = touch * _extent(xy) / np.sqrt(n)
|
|
119
|
+
graph = radius_neighbors_graph(xy, radius=max(radius, 1e-12), mode="connectivity")
|
|
120
|
+
_, labels = connected_components(graph, directed=False)
|
|
121
|
+
return float(np.bincount(labels).max() / n)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _without_self(idx: np.ndarray, k: int) -> np.ndarray:
|
|
125
|
+
"""Each row's first ``k`` neighbours, with the row's own index removed wherever it appears."""
|
|
126
|
+
keep = idx != np.arange(len(idx))[:, None]
|
|
127
|
+
order = np.argsort(~keep, axis=1, kind="stable")
|
|
128
|
+
return np.take_along_axis(idx, order, axis=1)[:, :k]
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class _Reference:
|
|
132
|
+
"""Everything about the clone profiles that every layout is compared with, computed once."""
|
|
133
|
+
|
|
134
|
+
def __init__(self, features: np.ndarray, *, k_max: int, k_local: int = 30, n_pairs: int = 20000,
|
|
135
|
+
random_state: int = 0):
|
|
136
|
+
from sklearn.neighbors import NearestNeighbors
|
|
137
|
+
|
|
138
|
+
self.features = np.ascontiguousarray(features, dtype=np.float32)
|
|
139
|
+
n = len(self.features)
|
|
140
|
+
self.k_local = int(max(5, min(k_local, n // 4)))
|
|
141
|
+
# exact neighbours, self included as the first column, as UMAP expects them
|
|
142
|
+
k_all = int(min(max(k_max, self.k_local + 1, 16), n - 1))
|
|
143
|
+
dist, idx = NearestNeighbors(n_neighbors=k_all).fit(self.features).kneighbors(self.features)
|
|
144
|
+
self.knn_dist, self.knn_idx = dist.astype(np.float32), idx.astype(np.int64)
|
|
145
|
+
self.local_idx = _without_self(self.knn_idx, self.k_local)
|
|
146
|
+
rng = np.random.default_rng(random_state)
|
|
147
|
+
i, j = rng.integers(0, n, n_pairs), rng.integers(0, n, n_pairs)
|
|
148
|
+
keep = i != j
|
|
149
|
+
self.pairs = (i[keep], j[keep])
|
|
150
|
+
self.profile_dist = np.linalg.norm(self.features[i[keep]] - self.features[j[keep]], axis=1)
|
|
151
|
+
|
|
152
|
+
def intrinsic_coherence(self, k: int = 15) -> float:
|
|
153
|
+
"""Fraction of clones in the largest component of the mutual k-nearest-neighbour graph of profiles."""
|
|
154
|
+
import scipy.sparse as sp
|
|
155
|
+
from scipy.sparse.csgraph import connected_components
|
|
156
|
+
|
|
157
|
+
n = len(self.features)
|
|
158
|
+
nb = _without_self(self.knn_idx, int(min(k, self.knn_idx.shape[1] - 1)))
|
|
159
|
+
rows = np.repeat(np.arange(n), nb.shape[1])
|
|
160
|
+
g = sp.csr_matrix((np.ones(rows.size), (rows, nb.ravel())), shape=(n, n))
|
|
161
|
+
_, labels = connected_components(g.multiply(g.T), directed=False)
|
|
162
|
+
return float(np.bincount(labels).max() / n)
|
|
163
|
+
|
|
164
|
+
def layout(self, n_neighbors: int, min_dist: float, spread: float, seed: int, repulsion: float = 1.0,
|
|
165
|
+
n_epochs: int | None = None) -> np.ndarray:
|
|
166
|
+
import umap
|
|
167
|
+
|
|
168
|
+
k = int(max(2, min(n_neighbors, self.knn_idx.shape[1])))
|
|
169
|
+
with warnings.catch_warnings():
|
|
170
|
+
warnings.simplefilter("ignore")
|
|
171
|
+
model = umap.UMAP(n_neighbors=k, min_dist=float(min_dist), spread=spread, random_state=int(seed),
|
|
172
|
+
repulsion_strength=float(repulsion), n_epochs=n_epochs,
|
|
173
|
+
precomputed_knn=(self.knn_idx[:, :k].copy(), self.knn_dist[:, :k].copy()))
|
|
174
|
+
return model.fit_transform(self.features)
|
|
175
|
+
|
|
176
|
+
def quality(self, xy: np.ndarray, target: float, penalty: float) -> dict:
|
|
177
|
+
from scipy.stats import spearmanr
|
|
178
|
+
from sklearn.neighbors import NearestNeighbors
|
|
179
|
+
|
|
180
|
+
k = self.k_local
|
|
181
|
+
near = _without_self(NearestNeighbors(n_neighbors=k + 1).fit(xy).kneighbors(xy, return_distance=False), k)
|
|
182
|
+
local = float((near[:, :, None] == self.local_idx[:, None, :]).any(axis=2).mean())
|
|
183
|
+
i, j = self.pairs
|
|
184
|
+
glob = float(spearmanr(self.profile_dist, np.linalg.norm(xy[i] - xy[j], axis=1)).statistic)
|
|
185
|
+
coh = _coherence(xy)
|
|
186
|
+
score = 0.5 * (local + glob) - penalty * max(0.0, target - coh)
|
|
187
|
+
return {"score": score, "coherence": coh, "local_fidelity": local, "global_fidelity": glob}
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _target(ref: _Reference, coherence_target) -> float:
|
|
191
|
+
if isinstance(coherence_target, str):
|
|
192
|
+
if coherence_target != "auto":
|
|
193
|
+
raise ValueError("coherence_target must be 'auto' or a number between 0 and 1.")
|
|
194
|
+
return min(0.99, ref.intrinsic_coherence())
|
|
195
|
+
if not isinstance(coherence_target, Real) or not 0 < coherence_target <= 1:
|
|
196
|
+
raise ValueError("coherence_target must be 'auto' or a number between 0 and 1.")
|
|
197
|
+
return float(coherence_target)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def map_quality(features, xy, *, coherence_target: float | str = "auto", penalty: float = 5.0,
|
|
201
|
+
random_state: int = 0) -> dict:
|
|
202
|
+
"""Score a clone map: coherence, local and global fidelity, and the tuner's combined score.
|
|
203
|
+
|
|
204
|
+
Parameters
|
|
205
|
+
----------
|
|
206
|
+
features
|
|
207
|
+
Clone profiles, one row per clone (``adata.uns['threadfin']['profiles']['features']``).
|
|
208
|
+
xy
|
|
209
|
+
The map, one row per clone in the same order.
|
|
210
|
+
coherence_target, penalty
|
|
211
|
+
As in :func:`tune_clone_map`.
|
|
212
|
+
|
|
213
|
+
Returns
|
|
214
|
+
-------
|
|
215
|
+
``dict`` with ``coherence`` (fraction of clones in the largest piece), ``local_fidelity`` (30-NN
|
|
216
|
+
preservation), ``global_fidelity`` (Spearman correlation of distances), ``score`` and the
|
|
217
|
+
``coherence_target`` used.
|
|
218
|
+
"""
|
|
219
|
+
feats = np.asarray(features, dtype=float)
|
|
220
|
+
xy = np.asarray(xy, dtype=float)
|
|
221
|
+
if feats.shape[0] != xy.shape[0]:
|
|
222
|
+
raise ValueError("features and xy must have one row per clone, in the same order.")
|
|
223
|
+
if feats.shape[0] < 15:
|
|
224
|
+
raise ValueError(f"Need at least 15 clones to score a clone map, got {feats.shape[0]}.")
|
|
225
|
+
ref = _Reference(feats, k_max=31, random_state=random_state)
|
|
226
|
+
target = _target(ref, coherence_target)
|
|
227
|
+
return ref.quality(xy, target, penalty) | {"coherence_target": target}
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
# --------------------------------------------------------------------------- the search
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
@dataclass
|
|
234
|
+
class MapTuning:
|
|
235
|
+
"""Result of :func:`tune_clone_map`."""
|
|
236
|
+
|
|
237
|
+
n_neighbors: int
|
|
238
|
+
min_dist: float
|
|
239
|
+
repulsion_strength: float
|
|
240
|
+
score: float
|
|
241
|
+
coherence: float
|
|
242
|
+
local_fidelity: float
|
|
243
|
+
global_fidelity: float
|
|
244
|
+
coherence_target: float
|
|
245
|
+
method: str
|
|
246
|
+
n_layouts: int
|
|
247
|
+
seconds: float
|
|
248
|
+
history: pd.DataFrame = field(repr=False)
|
|
249
|
+
candidates: pd.DataFrame = field(repr=False)
|
|
250
|
+
|
|
251
|
+
def to_uns(self) -> dict:
|
|
252
|
+
"""Record for ``adata.uns`` (tables stay DataFrames, so the AnnData can be written to h5ad)."""
|
|
253
|
+
rec = {k: v for k, v in self.as_dict().items() if k not in ("history", "candidates")}
|
|
254
|
+
return rec | {"history": self.history.copy(), "candidates": self.candidates.copy()}
|
|
255
|
+
|
|
256
|
+
def as_dict(self) -> dict:
|
|
257
|
+
"""JSON-friendly record of the search."""
|
|
258
|
+
return {"n_neighbors": int(self.n_neighbors), "min_dist": float(self.min_dist),
|
|
259
|
+
"repulsion_strength": float(self.repulsion_strength), "score": float(self.score),
|
|
260
|
+
"coherence": float(self.coherence), "local_fidelity": float(self.local_fidelity),
|
|
261
|
+
"global_fidelity": float(self.global_fidelity), "coherence_target": float(self.coherence_target),
|
|
262
|
+
"method": self.method, "n_layouts": int(self.n_layouts), "seconds": float(self.seconds),
|
|
263
|
+
"history": self.history.to_dict(orient="list"),
|
|
264
|
+
"candidates": self.candidates.to_dict(orient="list")}
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _key(setting) -> tuple:
|
|
268
|
+
k, md, rep = setting
|
|
269
|
+
return int(k), round(float(md), 3), round(float(rep), 3)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _layout_quality(ref: _Reference, key: tuple, spread: float, n_epochs, target: float, penalty: float) -> dict:
|
|
273
|
+
"""One layout at ``key = (n_neighbors, min_dist, repulsion, seed)`` and its score (runs in a worker)."""
|
|
274
|
+
k, md, rep, seed = key
|
|
275
|
+
xy = ref.layout(k, md, spread, seed, repulsion=rep, n_epochs=n_epochs)
|
|
276
|
+
return ref.quality(xy, target, penalty)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def resolve_n_jobs(n_jobs) -> int:
|
|
280
|
+
"""``n_jobs`` as a positive number of processes (``-1``: every available core; ``None``: one)."""
|
|
281
|
+
import os
|
|
282
|
+
|
|
283
|
+
if n_jobs is None:
|
|
284
|
+
return 1
|
|
285
|
+
if isinstance(n_jobs, bool) or not isinstance(n_jobs, (int, np.integer)) or n_jobs == 0:
|
|
286
|
+
raise ValueError(f"n_jobs must be a positive integer, -1 (all cores) or None, got {n_jobs!r}.")
|
|
287
|
+
try:
|
|
288
|
+
available = len(os.sched_getaffinity(0))
|
|
289
|
+
except AttributeError: # macOS / Windows
|
|
290
|
+
available = os.cpu_count() or 1
|
|
291
|
+
return int(available + 1 + n_jobs) if n_jobs < 0 else int(min(n_jobs, available))
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
class _Search:
|
|
295
|
+
"""Lays clones out at requested settings, scores each layout once and keeps every result."""
|
|
296
|
+
|
|
297
|
+
def __init__(self, ref: _Reference, target: float, penalty: float, spread: float, n_epochs, n_jobs: int = 1):
|
|
298
|
+
self.ref, self.target, self.penalty, self.spread, self.n_epochs = ref, target, penalty, spread, n_epochs
|
|
299
|
+
self.n_jobs = int(n_jobs)
|
|
300
|
+
self.cache: dict = {}
|
|
301
|
+
self.rows: list = []
|
|
302
|
+
|
|
303
|
+
def evaluate(self, setting, seed: int, stage: str) -> dict:
|
|
304
|
+
self.evaluate_many([(setting, seed, stage)])
|
|
305
|
+
return self.cache[_key(setting) + (int(seed),)]
|
|
306
|
+
|
|
307
|
+
def evaluate_many(self, requests) -> None:
|
|
308
|
+
"""Lay out and score every requested ``(setting, seed, stage)`` not seen before.
|
|
309
|
+
|
|
310
|
+
With ``n_jobs > 1`` the layouts are computed in parallel processes. A layout depends only on its
|
|
311
|
+
setting and seed, and results are stored in the order requested, so the outcome is identical for
|
|
312
|
+
any ``n_jobs``.
|
|
313
|
+
"""
|
|
314
|
+
todo, planned = [], set()
|
|
315
|
+
for setting, seed, stage in requests:
|
|
316
|
+
key = _key(setting) + (int(seed),)
|
|
317
|
+
if key not in self.cache and key not in planned:
|
|
318
|
+
planned.add(key)
|
|
319
|
+
todo.append((key, stage))
|
|
320
|
+
if not todo:
|
|
321
|
+
return
|
|
322
|
+
args = (self.spread, self.n_epochs, self.target, self.penalty)
|
|
323
|
+
if self.n_jobs > 1 and len(todo) > 1:
|
|
324
|
+
from joblib import Parallel, delayed
|
|
325
|
+
|
|
326
|
+
with warnings.catch_warnings():
|
|
327
|
+
warnings.simplefilter("ignore")
|
|
328
|
+
results = Parallel(n_jobs=min(self.n_jobs, len(todo)))(
|
|
329
|
+
delayed(_layout_quality)(self.ref, key, *args) for key, _ in todo)
|
|
330
|
+
else:
|
|
331
|
+
results = [_layout_quality(self.ref, key, *args) for key, _ in todo]
|
|
332
|
+
for (key, stage), q in zip(todo, results):
|
|
333
|
+
self.cache[key] = q
|
|
334
|
+
self.rows.append(dict(stage=stage, n_neighbors=key[0], min_dist=key[1], repulsion_strength=key[2],
|
|
335
|
+
seed=key[3], **q))
|
|
336
|
+
|
|
337
|
+
def count(self, setting) -> int:
|
|
338
|
+
key = _key(setting)
|
|
339
|
+
return sum(1 for kk in self.cache if kk[:3] == key)
|
|
340
|
+
|
|
341
|
+
def robust(self, setting) -> dict:
|
|
342
|
+
"""Mean fidelity and worst coherence over every layout of a setting, and the resulting score."""
|
|
343
|
+
key = _key(setting)
|
|
344
|
+
qs = pd.DataFrame([q for kk, q in self.cache.items() if kk[:3] == key])
|
|
345
|
+
local, glob = float(qs["local_fidelity"].mean()), float(qs["global_fidelity"].mean())
|
|
346
|
+
coh = float(qs["coherence"].min())
|
|
347
|
+
score = 0.5 * (local + glob) - self.penalty * max(0.0, self.target - coh)
|
|
348
|
+
return dict(n_neighbors=key[0], min_dist=key[1], repulsion_strength=key[2], n_layouts=len(qs),
|
|
349
|
+
score=score, coherence=coh, coherence_mean=float(qs["coherence"].mean()),
|
|
350
|
+
local_fidelity=local, global_fidelity=glob)
|
|
351
|
+
|
|
352
|
+
def seen(self) -> pd.DataFrame:
|
|
353
|
+
keys = sorted({kk[:3] for kk in self.cache})
|
|
354
|
+
return pd.DataFrame([self.robust(k) for k in keys])
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _smoothed(search: _Search, ks, mds, sigma: float) -> pd.DataFrame:
|
|
358
|
+
"""Robust score of every ladder setting (all its layouts so far), averaged with its ladder neighbours."""
|
|
359
|
+
from scipy.ndimage import gaussian_filter
|
|
360
|
+
|
|
361
|
+
raw = np.array([[search.robust((k, md, 1.0))["score"] for md in mds] for k in ks])
|
|
362
|
+
# normalised smoothing: each setting is averaged with the ladder neighbours that exist (no padding)
|
|
363
|
+
smooth = gaussian_filter(raw, sigma, mode="constant") / gaussian_filter(np.ones_like(raw), sigma, mode="constant")
|
|
364
|
+
return pd.DataFrame([(int(k), round(float(md), 3), raw[i, j], smooth[i, j]) for i, k in enumerate(ks)
|
|
365
|
+
for j, md in enumerate(mds)], columns=["n_neighbors", "min_dist", "score", "smoothed"])
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _ladder_candidates(search: _Search, ks, mds, rng, n_seeds: int, top: int, sigma: float, verbose: bool):
|
|
369
|
+
"""Steps 1-2: every ladder setting with shared seeds, worst-case coherence, Gaussian smoothing."""
|
|
370
|
+
seeds = [int(rng.integers(2**31 - 1)) for _ in range(n_seeds)]
|
|
371
|
+
search.evaluate_many([((k, md, 1.0), s, "ladder") for k in ks for md in mds for s in seeds])
|
|
372
|
+
tab = _smoothed(search, ks, mds, sigma)
|
|
373
|
+
for _, r in tab.nlargest(3, "smoothed").iterrows():
|
|
374
|
+
log(f" ladder: n_neighbors {int(r['n_neighbors'])}, min_dist {r['min_dist']:.2f} -> score "
|
|
375
|
+
f"{r['score']:.3f} (smoothed {r['smoothed']:.3f})", verbose)
|
|
376
|
+
picks = pd.concat([tab.nlargest(top, "smoothed"), tab.nlargest(2, "score")])
|
|
377
|
+
return list(dict.fromkeys((int(k), round(float(md), 3), 1.0)
|
|
378
|
+
for k, md in zip(picks["n_neighbors"], picks["min_dist"])))
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _adam_candidates(search: _Search, theta0, active, lo, hi, from_theta, rng, top: int, opts: dict,
|
|
382
|
+
verbose: bool):
|
|
383
|
+
"""Steps 1-2 for method='adam': a warm-up, then SPSA gradient estimates fed to Adam."""
|
|
384
|
+
n_iter, patience = opts.get("n_iter", 30), opts.get("patience", 8)
|
|
385
|
+
lr, c0 = opts.get("learning_rate", 0.25), opts.get("perturbation", 0.35)
|
|
386
|
+
b1, b2 = opts.get("betas", (0.8, 0.99))
|
|
387
|
+
target, theta = search.target, theta0.copy()
|
|
388
|
+
for _ in range(opts.get("max_warmup", 6)):
|
|
389
|
+
k_w, md_w = from_theta(theta)
|
|
390
|
+
for _ in range(2):
|
|
391
|
+
search.evaluate((k_w, md_w, 1.0), int(rng.integers(2**31 - 1)), "warm-up")
|
|
392
|
+
coh = search.robust((k_w, md_w, 1.0))["coherence"]
|
|
393
|
+
if coh >= target or (theta[active] >= hi[active] - 1e-9).all():
|
|
394
|
+
break
|
|
395
|
+
log(f" warm-up: n_neighbors {k_w}, min_dist {md_w:.2f} keeps {100 * coh:.1f}% of clones in one "
|
|
396
|
+
"piece; widening.", verbose)
|
|
397
|
+
theta = np.clip(theta + np.array([np.log(2.0), 1.0]) * active, lo, hi)
|
|
398
|
+
m, v = np.zeros(2), np.zeros(2)
|
|
399
|
+
best, stall = -np.inf, 0
|
|
400
|
+
for t in range(1, n_iter + 1):
|
|
401
|
+
c_t = c0 / t ** 0.101
|
|
402
|
+
delta = np.where(rng.random(2) < 0.5, -1.0, 1.0) * active
|
|
403
|
+
seed = int(rng.integers(2**31 - 1))
|
|
404
|
+
(k_p, md_p), (k_m, md_m) = from_theta(np.clip(theta + c_t * delta, lo, hi)), \
|
|
405
|
+
from_theta(np.clip(theta - c_t * delta, lo, hi))
|
|
406
|
+
s_p = search.evaluate((k_p, md_p, 1.0), seed, "adam")["score"]
|
|
407
|
+
s_m = search.evaluate((k_m, md_m, 1.0), seed, "adam")["score"]
|
|
408
|
+
# SPSA: one seed for both sides, so the difference is about the settings, not the seed
|
|
409
|
+
grad = np.zeros(2)
|
|
410
|
+
grad[active] = (s_p - s_m) / (2.0 * c_t * delta[active])
|
|
411
|
+
m = b1 * m + (1 - b1) * grad
|
|
412
|
+
v = b2 * v + (1 - b2) * grad ** 2
|
|
413
|
+
theta = np.clip(theta + lr * (m / (1 - b1 ** t)) / (np.sqrt(v / (1 - b2 ** t)) + 1e-8) * active, lo, hi)
|
|
414
|
+
log(f" Adam step {t:2d}: n_neighbors {k_p} / {k_m}, min_dist {md_p:.2f} / {md_m:.2f} -> score "
|
|
415
|
+
f"{s_p:.3f} / {s_m:.3f}", verbose)
|
|
416
|
+
if max(s_p, s_m) > best + 1e-3:
|
|
417
|
+
best, stall = max(s_p, s_m), 0
|
|
418
|
+
else:
|
|
419
|
+
stall += 1
|
|
420
|
+
if stall >= patience:
|
|
421
|
+
break
|
|
422
|
+
seen = search.seen().nlargest(top, "score")
|
|
423
|
+
return list(zip(seen["n_neighbors"].astype(int), seen["min_dist"].round(3), seen["repulsion_strength"]))
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _confirm(search: _Search, candidates, rng, n_layouts: int, tolerance: float, ladder=None):
|
|
427
|
+
"""Steps 3-4: each candidate to ``n_layouts`` layouts, then the choice.
|
|
428
|
+
|
|
429
|
+
Settings that held together in every layout are preferred. With a ladder (``(ks, mds, sigma)``), the
|
|
430
|
+
choice uses the smoothed score, recomputed with every layout so far, so that a setting surrounded by
|
|
431
|
+
settings that break apart loses to one in a robust neighbourhood. Ties go to the more coherent and
|
|
432
|
+
then to the standard (stronger) repulsion.
|
|
433
|
+
"""
|
|
434
|
+
requests, planned = [], {}
|
|
435
|
+
for setting in candidates:
|
|
436
|
+
key = _key(setting)
|
|
437
|
+
need = max(0, n_layouts - search.count(setting) - planned.get(key, 0))
|
|
438
|
+
planned[key] = planned.get(key, 0) + need
|
|
439
|
+
requests += [(setting, int(rng.integers(2**31 - 1)), "confirm") for _ in range(need)]
|
|
440
|
+
search.evaluate_many(requests)
|
|
441
|
+
tab = pd.DataFrame([search.robust(s) for s in candidates])
|
|
442
|
+
tab["smoothed"] = tab["score"]
|
|
443
|
+
if ladder is not None:
|
|
444
|
+
sm = _smoothed(search, *ladder).set_index(["n_neighbors", "min_dist"])["smoothed"]
|
|
445
|
+
std = tab["repulsion_strength"] == 1.0
|
|
446
|
+
idx = list(zip(tab.loc[std, "n_neighbors"], tab.loc[std, "min_dist"]))
|
|
447
|
+
tab.loc[std, "smoothed"] = sm.reindex(idx).to_numpy()
|
|
448
|
+
tab["smoothed"] = tab["smoothed"].fillna(tab["score"])
|
|
449
|
+
tab = tab.sort_values("smoothed", ascending=False).reset_index(drop=True)
|
|
450
|
+
pool = tab[tab["coherence"] >= search.target]
|
|
451
|
+
pool = pool if len(pool) else tab
|
|
452
|
+
tied = pool[pool["smoothed"] >= pool["smoothed"].max() - tolerance]
|
|
453
|
+
win = tied.sort_values(["coherence", "repulsion_strength", "smoothed"], ascending=False).iloc[0]
|
|
454
|
+
return tab, win
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _most_coherent(search: _Search, exclude, n: int, repulsion: float = 1.0):
|
|
458
|
+
"""The ``n`` settings with the highest worst-case coherence so far, other than those in ``exclude``."""
|
|
459
|
+
seen = search.seen()
|
|
460
|
+
seen = seen[seen["repulsion_strength"] == repulsion]
|
|
461
|
+
done = {_key(s) for s in exclude}
|
|
462
|
+
keep = [_key((k, md, rep)) not in done
|
|
463
|
+
for k, md, rep in zip(seen["n_neighbors"], seen["min_dist"], seen["repulsion_strength"])]
|
|
464
|
+
seen = seen[keep].sort_values(["coherence", "score"], ascending=False).head(n)
|
|
465
|
+
return [_key((k, md, rep)) for k, md, rep in zip(seen["n_neighbors"], seen["min_dist"], seen["repulsion_strength"])]
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def tune_clone_map(
|
|
469
|
+
features,
|
|
470
|
+
*,
|
|
471
|
+
n_neighbors: int | str = "auto",
|
|
472
|
+
min_dist: float | str = "auto",
|
|
473
|
+
spread: float = 1.0,
|
|
474
|
+
method: str = "ladder",
|
|
475
|
+
coherence_target: float | str = "auto",
|
|
476
|
+
penalty: float = 5.0,
|
|
477
|
+
tolerance: float = 0.005,
|
|
478
|
+
ladder_seeds: int = 2,
|
|
479
|
+
top: int = 4,
|
|
480
|
+
n_layouts: int = 6,
|
|
481
|
+
rescue: int = 3,
|
|
482
|
+
allow_weaker_repulsion: bool = True,
|
|
483
|
+
smoothing: float = 0.7,
|
|
484
|
+
search_epochs: int | None = 200,
|
|
485
|
+
max_clones: int | None = 3000,
|
|
486
|
+
full_layouts: int = 2,
|
|
487
|
+
adam_options: dict | None = None,
|
|
488
|
+
random_state: int = 0,
|
|
489
|
+
n_jobs: int | None = 1,
|
|
490
|
+
verbose: bool = True,
|
|
491
|
+
) -> MapTuning:
|
|
492
|
+
"""Choose UMAP ``n_neighbors`` and ``min_dist`` for a clone map.
|
|
493
|
+
|
|
494
|
+
Looks for the most faithful layout that keeps at least ``coherence_target`` of the clones in one
|
|
495
|
+
connected piece in every layout tried, which makes the choice robust to the randomness of UMAP (see
|
|
496
|
+
the module documentation for the score and the search). A setting passed as a number is held fixed
|
|
497
|
+
and only the other is tuned.
|
|
498
|
+
|
|
499
|
+
Parameters
|
|
500
|
+
----------
|
|
501
|
+
features
|
|
502
|
+
Clone profiles, one row per clone.
|
|
503
|
+
n_neighbors, min_dist
|
|
504
|
+
``"auto"`` to tune the setting, or a number to hold it fixed.
|
|
505
|
+
spread
|
|
506
|
+
UMAP ``spread``, held fixed.
|
|
507
|
+
method
|
|
508
|
+
``"ladder"`` (default; a fixed ladder of settings, smoothed and confirmed) or ``"adam"``
|
|
509
|
+
(stochastic-gradient search; options in ``adam_options``).
|
|
510
|
+
coherence_target
|
|
511
|
+
Fraction of clones that must lie in one connected piece of the map. ``"auto"`` (default) takes
|
|
512
|
+
it from the profiles: the fraction of clones in the largest component of their mutual
|
|
513
|
+
15-nearest-neighbour graph, capped at 0.99, so the map is not more fragmented than the profiles.
|
|
514
|
+
Pass a number (e.g. 0.99 or 1.0) to demand a single piece regardless.
|
|
515
|
+
penalty
|
|
516
|
+
Score lost per unit of coherence below the target. The default (5) makes the target binding:
|
|
517
|
+
one percentage point of coherence outweighs any realistic gain in fidelity.
|
|
518
|
+
tolerance
|
|
519
|
+
Confirmed settings whose score is within this distance of the best are treated as equally
|
|
520
|
+
faithful, and the most coherent of them is chosen (seed-to-seed noise is about 0.005).
|
|
521
|
+
ladder_seeds, top, n_layouts
|
|
522
|
+
Layouts per ladder setting; how many of the best settings are confirmed; how many layouts each
|
|
523
|
+
confirmed setting is scored on in total.
|
|
524
|
+
rescue, allow_weaker_repulsion
|
|
525
|
+
When the winner falls short of the target, how many of the most coherent settings seen are also
|
|
526
|
+
confirmed, and whether weaker repulsion may then be tried.
|
|
527
|
+
smoothing
|
|
528
|
+
Width (in ladder steps) of the Gaussian that averages each ladder setting with its neighbours.
|
|
529
|
+
search_epochs
|
|
530
|
+
UMAP optimisation epochs for layouts during the search (``None``: UMAP's default).
|
|
531
|
+
max_clones, full_layouts
|
|
532
|
+
Above ``max_clones`` clones, the search runs on a random subset of this size, ``n_neighbors`` is
|
|
533
|
+
scaled up to the full data in proportion to the number of clones, and the best settings are then
|
|
534
|
+
confirmed on every clone with ``full_layouts`` layouts each. ``None`` always searches on every
|
|
535
|
+
clone.
|
|
536
|
+
adam_options
|
|
537
|
+
For ``method="adam"``: ``n_iter`` (30), ``patience`` (8), ``learning_rate`` (0.25),
|
|
538
|
+
``perturbation`` (0.35), ``betas`` ((0.8, 0.99)), ``max_warmup`` (6) and ``start`` (default
|
|
539
|
+
``sqrt(N)`` neighbours between 15 and 100, and ``min_dist`` 0.3).
|
|
540
|
+
random_state
|
|
541
|
+
Seed for layout seeds and the clone subset; the result is reproducible for a given seed.
|
|
542
|
+
n_jobs
|
|
543
|
+
Processes used to compute layouts in parallel (``-1``: all available cores). Every layout depends
|
|
544
|
+
only on its setting and seed, so the chosen settings are identical for any ``n_jobs``; only the
|
|
545
|
+
time changes.
|
|
546
|
+
|
|
547
|
+
Returns
|
|
548
|
+
-------
|
|
549
|
+
:class:`MapTuning` with the chosen settings (including ``repulsion_strength``, 1.0 unless the rescue
|
|
550
|
+
needed weaker repulsion), their confirmed scores, every layout scored (``history``) and the confirmed
|
|
551
|
+
candidates (``candidates``).
|
|
552
|
+
"""
|
|
553
|
+
t0 = time.time()
|
|
554
|
+
feats = np.asarray(features, dtype=float)
|
|
555
|
+
n_all = feats.shape[0]
|
|
556
|
+
if n_all < 15:
|
|
557
|
+
raise ValueError(f"Need at least 15 clones to tune a clone map, got {n_all}.")
|
|
558
|
+
if method not in {"ladder", "adam"}:
|
|
559
|
+
raise ValueError("method must be 'ladder' or 'adam'.")
|
|
560
|
+
for name, value in (("n_neighbors", n_neighbors), ("min_dist", min_dist)):
|
|
561
|
+
if isinstance(value, str) and value != "auto":
|
|
562
|
+
raise ValueError(f"{name} must be 'auto' or a number, got {value!r}.")
|
|
563
|
+
tune_k, tune_md = n_neighbors == "auto", min_dist == "auto"
|
|
564
|
+
if not (tune_k or tune_md):
|
|
565
|
+
raise ValueError("Nothing to tune: pass n_neighbors='auto' and/or min_dist='auto'.")
|
|
566
|
+
rng = np.random.default_rng(random_state)
|
|
567
|
+
|
|
568
|
+
# search on a random subset when there are many clones; neighbour counts are scaled back up
|
|
569
|
+
feats_all, scale = feats, 1.0
|
|
570
|
+
if max_clones is not None and n_all > max_clones:
|
|
571
|
+
feats = feats[np.sort(rng.choice(n_all, size=int(max_clones), replace=False))]
|
|
572
|
+
scale = n_all / feats.shape[0]
|
|
573
|
+
n = feats.shape[0]
|
|
574
|
+
k_lo = N_NEIGHBORS_RANGE[0]
|
|
575
|
+
k_cap = int(max(k_lo, min(N_NEIGHBORS_RANGE[1], n - 1, n // 2 if n >= 30 else n - 1)))
|
|
576
|
+
|
|
577
|
+
ref = _Reference(feats, k_max=k_cap, random_state=int(rng.integers(2**31 - 1)))
|
|
578
|
+
target = _target(ref, coherence_target)
|
|
579
|
+
n_jobs = resolve_n_jobs(n_jobs)
|
|
580
|
+
search = _Search(ref, target, penalty, spread, search_epochs, n_jobs)
|
|
581
|
+
k_fixed = int(np.clip(round(n_neighbors / scale), k_lo, k_cap)) if not tune_k else None
|
|
582
|
+
log(f"choosing clone-map settings ({method}) on {n:,} clones"
|
|
583
|
+
f"{f' (random subset of {n_all:,})' if scale > 1 else ''}; target: at least {100 * target:.1f}% of "
|
|
584
|
+
"clones in one piece.", verbose)
|
|
585
|
+
|
|
586
|
+
if method == "ladder":
|
|
587
|
+
ks = [k for k in N_NEIGHBORS_LADDER if k <= k_cap] or [k_cap]
|
|
588
|
+
if len(ks) < 2:
|
|
589
|
+
ks = sorted({k_lo, k_cap})
|
|
590
|
+
ks = ks if tune_k else [k_fixed]
|
|
591
|
+
mds = list(MIN_DIST_LADDER) if tune_md else [float(min_dist)]
|
|
592
|
+
candidates = _ladder_candidates(search, ks, mds, rng, ladder_seeds, top, smoothing, verbose)
|
|
593
|
+
ladder = (ks, mds, smoothing)
|
|
594
|
+
else:
|
|
595
|
+
ladder = None
|
|
596
|
+
opts = dict(adam_options or {})
|
|
597
|
+
md_hi = MIN_DIST_RANGE[1]
|
|
598
|
+
lo = np.array([np.log(k_lo), -4.0])
|
|
599
|
+
hi = np.array([np.log(k_cap), 4.0])
|
|
600
|
+
active = np.array([tune_k, tune_md])
|
|
601
|
+
start = opts.get("start") or (int(np.clip(round(np.sqrt(n_all)), 15, 100)), 0.3)
|
|
602
|
+
k0 = start[0] / scale if tune_k else k_fixed
|
|
603
|
+
md0 = start[1] if tune_md else float(min_dist)
|
|
604
|
+
|
|
605
|
+
def from_theta(th):
|
|
606
|
+
k = float(np.exp(th[0])) if tune_k else k_fixed
|
|
607
|
+
md = md_hi / (1.0 + np.exp(-th[1])) if tune_md else float(min_dist)
|
|
608
|
+
return int(np.clip(round(k), k_lo, k_cap)), round(float(md), 3)
|
|
609
|
+
|
|
610
|
+
p = np.clip(md0, 0.005, md_hi - 0.005) / md_hi
|
|
611
|
+
theta0 = np.clip(np.array([np.log(np.clip(k0, k_lo, k_cap)), np.log(p / (1 - p))]), lo, hi)
|
|
612
|
+
candidates = _adam_candidates(search, theta0, active, lo, hi, from_theta, rng, top, opts, verbose)
|
|
613
|
+
|
|
614
|
+
table, win = _confirm(search, candidates, rng, n_layouts, tolerance, ladder)
|
|
615
|
+
if win["coherence"] < target and rescue > 0:
|
|
616
|
+
extra = _most_coherent(search, candidates, rescue)
|
|
617
|
+
if extra:
|
|
618
|
+
log(f" the best setting broke apart in one of its layouts; confirming {len(extra)} more "
|
|
619
|
+
"coherent settings.", verbose)
|
|
620
|
+
candidates = candidates + extra
|
|
621
|
+
table, win = _confirm(search, candidates, rng, n_layouts, tolerance, ladder)
|
|
622
|
+
if win["coherence"] < target and allow_weaker_repulsion:
|
|
623
|
+
base = _most_coherent(search, [], max(rescue, 1)) + [_key(s) for s in candidates[:2]]
|
|
624
|
+
for rep in REPULSION_RESCUE:
|
|
625
|
+
weaker = list(dict.fromkeys((k, md, rep) for k, md, _ in base))
|
|
626
|
+
log(f" no setting held together in every layout; trying weaker repulsion ({rep}).", verbose)
|
|
627
|
+
candidates = candidates + weaker
|
|
628
|
+
table, win = _confirm(search, candidates, rng, n_layouts, tolerance, ladder)
|
|
629
|
+
if win["coherence"] >= target:
|
|
630
|
+
break
|
|
631
|
+
if win["coherence"] < target:
|
|
632
|
+
log(f" warning: no setting kept {100 * target:.1f}% of clones in one piece in every layout; the "
|
|
633
|
+
f"most coherent one reached {100 * win['coherence']:.1f}%.", verbose)
|
|
634
|
+
|
|
635
|
+
k_full = lambda k: int(np.clip(round(k * scale), k_lo, min(N_NEIGHBORS_RANGE[1], n_all - 1))) # noqa: E731
|
|
636
|
+
rows = list(search.rows)
|
|
637
|
+
if scale > 1:
|
|
638
|
+
# a setting that holds together on the subset need not do so on every clone: the best settings are
|
|
639
|
+
# confirmed again on the full data, with neighbour counts scaled up, before one is chosen
|
|
640
|
+
full_ref = _Reference(feats_all, k_max=min(N_NEIGHBORS_RANGE[1], n_all - 1),
|
|
641
|
+
random_state=int(rng.integers(2**31 - 1)))
|
|
642
|
+
full_target = _target(full_ref, coherence_target)
|
|
643
|
+
full = _Search(full_ref, full_target, penalty, spread, search_epochs, n_jobs)
|
|
644
|
+
ranked = table.sort_values(["coherence", "smoothed"], ascending=False)
|
|
645
|
+
picks = [(int(n_neighbors) if not tune_k else k_full(k), float(md), float(rep))
|
|
646
|
+
for k, md, rep in zip(ranked["n_neighbors"], ranked["min_dist"], ranked["repulsion_strength"])]
|
|
647
|
+
picks = list(dict.fromkeys(picks))[:max(top, 2)]
|
|
648
|
+
log(f" confirming the {len(picks)} best settings on all {n_all:,} clones.", verbose)
|
|
649
|
+
table, win = _confirm(full, picks, rng, full_layouts, tolerance)
|
|
650
|
+
if win["coherence"] < full_target and tune_md:
|
|
651
|
+
# rescue on all clones: the widest spacings, where weakly connected groups stay attached
|
|
652
|
+
ladder_k = [k for k in N_NEIGHBORS_LADDER if k <= k_cap] or [k_cap]
|
|
653
|
+
ks_full = sorted({int(n_neighbors) if not tune_k else k_full(k) for k in ladder_k})
|
|
654
|
+
corner = [(k, md, 1.0) for k in ks_full for md in MIN_DIST_LADDER[-2:]]
|
|
655
|
+
corner = [c for c in corner if _key(c) not in {_key(p) for p in picks}]
|
|
656
|
+
log(f" no setting held together on all clones; scanning {len(corner)} wide-spacing settings.",
|
|
657
|
+
verbose)
|
|
658
|
+
seed = int(rng.integers(2**31 - 1))
|
|
659
|
+
full.evaluate_many([(c, seed, "full data") for c in corner])
|
|
660
|
+
corner.sort(key=lambda c: (full.robust(c)["coherence"], full.robust(c)["score"]), reverse=True)
|
|
661
|
+
picks = picks + corner[:3]
|
|
662
|
+
table, win = _confirm(full, picks, rng, full_layouts, tolerance)
|
|
663
|
+
if win["coherence"] < full_target and allow_weaker_repulsion:
|
|
664
|
+
for rep in REPULSION_RESCUE:
|
|
665
|
+
coherent_first = table.sort_values(["coherence", "score"], ascending=False)
|
|
666
|
+
base = list(zip(coherent_first["n_neighbors"].astype(int), coherent_first["min_dist"]))[:2]
|
|
667
|
+
weaker = list(dict.fromkeys((int(k), float(md), rep) for k, md in base))
|
|
668
|
+
log(f" no setting held together on all clones; trying weaker repulsion ({rep}).", verbose)
|
|
669
|
+
picks = picks + weaker
|
|
670
|
+
table, win = _confirm(full, picks, rng, full_layouts, tolerance)
|
|
671
|
+
if win["coherence"] >= full_target:
|
|
672
|
+
break
|
|
673
|
+
rows += [r | {"stage": "full data"} for r in full.rows]
|
|
674
|
+
target, scale = full_target, 1.0
|
|
675
|
+
if win["coherence"] < target:
|
|
676
|
+
log(f" warning: on all clones, the most coherent setting kept {100 * win['coherence']:.1f}% of "
|
|
677
|
+
f"clones in one piece (target {100 * target:.1f}%).", verbose)
|
|
678
|
+
k_best = int(n_neighbors) if not tune_k else k_full(win["n_neighbors"])
|
|
679
|
+
table["n_neighbors_full_data"] = [int(n_neighbors) if not tune_k else k_full(k) for k in table["n_neighbors"]]
|
|
680
|
+
result = MapTuning(
|
|
681
|
+
n_neighbors=k_best, min_dist=float(win["min_dist"]), repulsion_strength=float(win["repulsion_strength"]),
|
|
682
|
+
score=float(win["score"]), coherence=float(win["coherence"]), local_fidelity=float(win["local_fidelity"]),
|
|
683
|
+
global_fidelity=float(win["global_fidelity"]), coherence_target=target, method=method,
|
|
684
|
+
n_layouts=len(rows), seconds=time.time() - t0, history=pd.DataFrame(rows), candidates=table)
|
|
685
|
+
rep_txt = "" if result.repulsion_strength == 1.0 else f", repulsion_strength {result.repulsion_strength:g}"
|
|
686
|
+
log(f"clone-map settings: n_neighbors {result.n_neighbors}, min_dist {result.min_dist:.2f}{rep_txt}; "
|
|
687
|
+
f"in every layout at least {100 * result.coherence:.1f}% of clones form one piece (local fidelity "
|
|
688
|
+
f"{result.local_fidelity:.3f}, global fidelity {result.global_fidelity:.3f}; {result.n_layouts} "
|
|
689
|
+
f"layouts, {result.seconds:.0f} s).", verbose)
|
|
690
|
+
return result
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
# --------------------------------------------------------------------------- the map that is returned
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def layout_clone_map(features, *, n_neighbors: int, min_dist: float, repulsion_strength: float = 1.0,
|
|
697
|
+
spread: float = 1.0, coherence_target: float | str = "auto", tuning: MapTuning | None = None,
|
|
698
|
+
max_retries: int = 4, random_state: int = 0):
|
|
699
|
+
"""Lay the clones out at given settings, at UMAP's default number of epochs, and check the result.
|
|
700
|
+
|
|
701
|
+
When the settings came from :func:`tune_clone_map` (``tuning``) and this layout falls short of the
|
|
702
|
+
coherence target, up to ``max_retries`` further layouts are tried: two more seeds, then the next
|
|
703
|
+
confirmed settings that held together in every layout of the search. The most coherent layout is
|
|
704
|
+
returned. With settings given by hand, the layout is returned as it is.
|
|
705
|
+
|
|
706
|
+
Returns
|
|
707
|
+
-------
|
|
708
|
+
``(xy, quality, setting, attempts)``: the coordinates, :func:`map_quality` of the returned layout,
|
|
709
|
+
the ``(n_neighbors, min_dist, repulsion_strength, seed)`` that produced it, and every layout tried.
|
|
710
|
+
"""
|
|
711
|
+
import umap
|
|
712
|
+
|
|
713
|
+
feats = np.asarray(features, dtype=float)
|
|
714
|
+
n = feats.shape[0]
|
|
715
|
+
target = tuning.coherence_target if tuning is not None else coherence_target
|
|
716
|
+
|
|
717
|
+
def run(k, md, rep, seed):
|
|
718
|
+
with warnings.catch_warnings():
|
|
719
|
+
warnings.filterwarnings("ignore", message=".*n_jobs value.*")
|
|
720
|
+
xy = umap.UMAP(n_neighbors=int(min(k, n - 1)), min_dist=float(md), spread=spread,
|
|
721
|
+
repulsion_strength=float(rep), random_state=int(seed)).fit_transform(feats)
|
|
722
|
+
return xy, map_quality(feats, xy, coherence_target=target)
|
|
723
|
+
|
|
724
|
+
setting = (int(n_neighbors), float(min_dist), float(repulsion_strength), int(random_state))
|
|
725
|
+
xy, quality = run(*setting)
|
|
726
|
+
names = ("n_neighbors", "min_dist", "repulsion_strength", "seed")
|
|
727
|
+
attempts = [dict(zip(names, setting), coherence=quality["coherence"], score=quality["score"])]
|
|
728
|
+
if tuning is not None and quality["coherence"] < quality["coherence_target"] - 0.005:
|
|
729
|
+
tries = [setting[:3] + (int(random_state) + i,) for i in (1, 2)]
|
|
730
|
+
for _, r in tuning.candidates.iterrows():
|
|
731
|
+
alt = (int(r["n_neighbors_full_data"]), float(r["min_dist"]), float(r["repulsion_strength"]))
|
|
732
|
+
if alt != setting[:3] and r["coherence"] >= tuning.coherence_target:
|
|
733
|
+
tries.append(alt + (int(random_state),))
|
|
734
|
+
best = (quality["coherence"], quality["score"], xy, quality, setting)
|
|
735
|
+
for trial in tries[:max_retries]:
|
|
736
|
+
xy_t, q_t = run(*trial)
|
|
737
|
+
attempts.append(dict(zip(names, trial), coherence=q_t["coherence"], score=q_t["score"]))
|
|
738
|
+
if (q_t["coherence"], q_t["score"]) > best[:2]:
|
|
739
|
+
best = (q_t["coherence"], q_t["score"], xy_t, q_t, trial)
|
|
740
|
+
if q_t["coherence"] >= q_t["coherence_target"] - 0.005:
|
|
741
|
+
break
|
|
742
|
+
_, _, xy, quality, setting = best
|
|
743
|
+
return xy, quality, setting, attempts
|