DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,599 @@
1
+ import dataclasses
2
+ import time
3
+ import warnings
4
+ from logging import getLogger
5
+ from pathlib import Path
6
+ from typing import Any, Generator, cast
7
+
8
+ import numpy as np
9
+ import torch
10
+ from probeinterface import Probe
11
+ from scipy.spatial import KDTree
12
+ from scipy.spatial.distance import pdist
13
+ from spikeinterface.generation.drift_tools import (
14
+ DriftingTemplates,
15
+ InjectDriftingTemplatesRecording,
16
+ move_dense_templates,
17
+ )
18
+ from tqdm.auto import tqdm
19
+
20
+ from ..config import DeveloperConfig
21
+ from ..templates import TemplateData
22
+ from ..util.data_util import DARTsortSorting
23
+ from ..util.internal_config import ComputationConfig, unshifted_raw_template_cfg
24
+ from ..util.py_util import resolve_path
25
+ from . import analysis, comparison, simkit
26
+
27
+ logger = getLogger(__name__)
28
+
29
+
30
+ def get_drifty_hybrid_recording(
31
+ recording,
32
+ templates,
33
+ motion_estimate,
34
+ sorting=None,
35
+ displacement_sampling_frequency=5.0,
36
+ seed=0,
37
+ firing_rates=None,
38
+ peak_channels=None,
39
+ amplitude_scale_std=0.1,
40
+ amplitude_factor=None,
41
+ ):
42
+ """
43
+ :param: recording
44
+ :param: templates object
45
+ :param: motion estimate object
46
+ :param: firing_rates
47
+ :param: peak_channels
48
+ :param: amplitude_scale_std -- std of gamma distributed amplitude variation if
49
+ amplitude_factor is None
50
+ :param: amplitude_factor array of length n_spikes with amplitude factors
51
+ """
52
+ num_units = templates.num_units
53
+ rg = np.random.default_rng(seed=seed)
54
+
55
+ if peak_channels is None:
56
+ (central_disp_index,) = np.flatnonzero(np.all(templates.displacements == 0, 1))
57
+ central_templates = templates.templates_array_moved[central_disp_index]
58
+ peak_channels = np.ptp(central_templates, 1).argmax(1)
59
+
60
+ if sorting is None:
61
+ sorting = simkit.simulate_sorting(
62
+ num_units,
63
+ recording.get_num_samples(),
64
+ firing_rates=firing_rates,
65
+ rg=rg,
66
+ spike_length_samples=templates.num_samples,
67
+ sampling_frequency=recording.sampling_frequency,
68
+ )
69
+ n_spikes = sorting.count_total_num_spikes()
70
+
71
+ # Default amplitude scalings for spikes drawn from gamma
72
+ if amplitude_factor is None:
73
+ if amplitude_scale_std:
74
+ shape = 1.0 / (amplitude_scale_std**1.5)
75
+ amplitude_factor = rg.gamma(shape, scale=1.0 / (shape - 1), size=n_spikes)
76
+ else:
77
+ amplitude_factor = np.ones(n_spikes)
78
+
79
+ depths = recording.get_probe().contact_positions[:, 1][peak_channels]
80
+ t_start = recording.sample_index_to_time(0)
81
+ t_end = recording.sample_index_to_time(recording.get_num_samples() - 1)
82
+ motion_times_s = np.arange(
83
+ t_start, t_end, step=1.0 / displacement_sampling_frequency
84
+ )
85
+
86
+ disp_y = motion_estimate.disp_at_s(motion_times_s, depths, grid=True)
87
+
88
+ disp = np.zeros((motion_times_s.shape[0], 2, num_units))
89
+ disp[:, 1, :] = disp_y.T
90
+ # this tricks SI into using one displacement per unit
91
+ displacement_unit_factor = np.eye(num_units)
92
+
93
+ if not sorting.check_serializability(type="json"):
94
+ warnings.warn(
95
+ "Your sorting is not serializable, which could lead to problems later."
96
+ )
97
+
98
+ rec = InjectDriftingTemplatesRecording(
99
+ sorting=sorting,
100
+ drifting_templates=templates,
101
+ parent_recording=recording,
102
+ displacement_vectors=[disp],
103
+ displacement_sampling_frequency=displacement_sampling_frequency,
104
+ displacement_unit_factor=displacement_unit_factor,
105
+ amplitude_factor=amplitude_factor,
106
+ )
107
+ rec.annotate(peak_channel=peak_channels.tolist())
108
+ return rec
109
+
110
+
111
+ def precompute_displaced_registered_templates(
112
+ template_data: TemplateData,
113
+ geometry: np.ndarray,
114
+ displacements: np.ndarray,
115
+ sampling_frequency: float = 30000,
116
+ template_subset=slice(None),
117
+ ) -> DriftingTemplates:
118
+ """Use spikeinterface tools to turn templates on registered geom into
119
+ precomputed drifting templates on the regular geom.
120
+ """
121
+ assert template_data.registered_geom is not None
122
+ source_probe = Probe(ndim=template_data.registered_geom.ndim)
123
+ source_probe.set_contacts(positions=template_data.registered_geom)
124
+ target_probe = Probe(ndim=geometry.ndim)
125
+ target_probe.set_contacts(positions=geometry)
126
+
127
+ shifted_templates = move_dense_templates(
128
+ templates_array=template_data.templates[template_subset],
129
+ displacements=displacements,
130
+ source_probe=source_probe,
131
+ dest_probe=target_probe,
132
+ )
133
+
134
+ ret = DriftingTemplates.from_precomputed_templates(
135
+ templates_array_moved=shifted_templates,
136
+ displacements=displacements,
137
+ sampling_frequency=sampling_frequency,
138
+ nbefore=template_data.trough_offset_samples,
139
+ probe=target_probe,
140
+ )
141
+ return ret
142
+
143
+
144
+ def closest_clustering(
145
+ gt_st, peel_st, geom=None, match_dt_ms=0.1, match_radius_um=0.0, p=2.0
146
+ ):
147
+ frames_per_ms = gt_st.sampling_frequency / 1000
148
+ delta_frames = match_dt_ms * frames_per_ms
149
+ rescale = [delta_frames]
150
+ gt_pos = gt_st.times_samples[:, None]
151
+ peel_pos = peel_st.times_samples[:, None]
152
+ if match_radius_um:
153
+ assert isinstance(geom, np.ndarray)
154
+ rescale = rescale + (geom.shape[1] * [match_radius_um])
155
+ gt_pos = np.c_[gt_pos, geom[gt_st.channels]]
156
+ peel_pos = np.c_[peel_pos, geom[peel_st.channels]]
157
+ else:
158
+ gt_pos = gt_pos.astype(float)
159
+ peel_pos = peel_pos.astype(float)
160
+ gt_pos /= rescale
161
+ peel_pos /= rescale
162
+ peelix2gtix = greedy_match(gt_pos, peel_pos, dx=1.0 / frames_per_ms)
163
+ labels = peelix2gtix.copy() # type: ignore
164
+ labels[labels >= 0] = gt_st.labels[labels[labels >= 0]]
165
+
166
+ return peel_st.ephemeral_replace(labels=labels, gt_match_ix=peelix2gtix)
167
+
168
+
169
+ def greedy_match(
170
+ gt_coords,
171
+ test_coords,
172
+ max_val=1.0,
173
+ dx=1.0 / 30,
174
+ workers=-1,
175
+ p=2,
176
+ show_progress=True,
177
+ ):
178
+ """Greedily match spikes in a test sorting to those in a GT sorting
179
+
180
+ Iteratively expands a spatiotemporal neighborhood around each test spike
181
+ until a GT spike lands in the neighborhood; then arbitrarily pick from
182
+ the GT spikes in the neighborhood as a match.
183
+
184
+ Each spike in the GT and test spike trains will be matched to only one
185
+ partner in the other sorting.
186
+
187
+ Returns gt unit labels for each tested spike and an array describing
188
+ whether each gt spike was matched.
189
+
190
+ TODO: Be a little smarter. If there are ties, pick "best friend" unit.
191
+ Could be done with two iterations.
192
+
193
+ TODO: Return the gt->test assignments, which are easily computable here,
194
+ if needed.
195
+
196
+ Returns
197
+ -------
198
+ assignments: np.ndarray
199
+ GT spike ind for each tested spike
200
+ gt_unmatched: np.ndarray
201
+ Whether a GT spike had no tested match
202
+ """
203
+ assignments = np.full(len(test_coords), -1, dtype=np.intp)
204
+ gt_unmatched = np.ones(len(gt_coords), dtype=bool)
205
+
206
+ thresholds = np.arange(0.0 + 1e-5, max_val + dx + 2e-5, dx)
207
+ if show_progress:
208
+ thresholds = tqdm(thresholds, desc="Greedy match")
209
+
210
+ for j, thresh in enumerate(thresholds):
211
+ test_unmatched = np.flatnonzero(assignments < 0)
212
+ if not test_unmatched.size:
213
+ break
214
+ test_kdtree = KDTree(test_coords[test_unmatched])
215
+ gt_ix = np.flatnonzero(gt_unmatched)
216
+ d, i = test_kdtree.query(
217
+ gt_coords[gt_ix],
218
+ k=1,
219
+ distance_upper_bound=min(thresh, max_val), # type: ignore
220
+ workers=workers,
221
+ p=p,
222
+ )
223
+ i = np.atleast_1d(i)
224
+ # handle multiple gt spikes getting matched to the same peel ix
225
+ thresh_matched = i < test_kdtree.n
226
+ _, ii = np.unique(i, return_index=True)
227
+ i = i[ii]
228
+ thresh_matched = thresh_matched[ii]
229
+
230
+ gt_ix = gt_ix[ii]
231
+ gt_ix = gt_ix[thresh_matched]
232
+ i = i[thresh_matched]
233
+ assignments[test_unmatched[i]] = gt_ix
234
+ gt_unmatched[gt_ix] = False
235
+
236
+ if not gt_unmatched.any():
237
+ break
238
+ if thresh > max_val:
239
+ break
240
+
241
+ return assignments, gt_unmatched
242
+
243
+
244
+ def greedy_match_counts(
245
+ gt_sorting,
246
+ tested_sorting,
247
+ radius_um=35.0,
248
+ radius_frames=12,
249
+ show_progress=True,
250
+ ):
251
+ """A greedy confusion matrix computed using greedy_match()."""
252
+ from scipy.optimize import linear_sum_assignment
253
+
254
+ gt_t = gt_sorting.times_samples / radius_frames
255
+ tested_t = tested_sorting.times_samples / radius_frames
256
+
257
+ geom = getattr(gt_sorting, "geom", getattr(tested_sorting, "geom", None))
258
+ assert geom is not None
259
+ gt_x = geom[gt_sorting.channels] / radius_um
260
+ tested_x = geom[tested_sorting.channels] / radius_um
261
+
262
+ step = min(1.0 / radius_frames, pdist(geom).min() / radius_um) / 2
263
+
264
+ test2gt_spike, gt_unmatched = greedy_match(
265
+ np.c_[gt_t, gt_x], np.c_[tested_t, tested_x], dx=step
266
+ )
267
+ counts = np.zeros(
268
+ (gt_sorting.unit_ids.max() + 1, tested_sorting.unit_ids.max() + 1),
269
+ dtype=np.int32,
270
+ )
271
+ test_matched_spike = np.flatnonzero(
272
+ np.logical_and(test2gt_spike >= 0, tested_sorting.labels >= 0)
273
+ )
274
+
275
+ matched_gt_labels = gt_sorting.labels[test2gt_spike[test_matched_spike]]
276
+ matched_test_labels = tested_sorting.labels[test_matched_spike]
277
+
278
+ np.add.at(counts, (matched_gt_labels, matched_test_labels), 1)
279
+
280
+ row_ind, col_ind = linear_sum_assignment(counts, maximize=True)
281
+
282
+ return dict(
283
+ counts=counts,
284
+ test2gt_spike=test2gt_spike,
285
+ gt_unmatched=gt_unmatched,
286
+ test_matched_spike=test_matched_spike,
287
+ matched_gt_labels=matched_gt_labels,
288
+ matched_test_labels=matched_test_labels,
289
+ counts_ord=counts[row_ind][:, col_ind],
290
+ gt_unit_order=row_ind,
291
+ test_unit_order=col_ind,
292
+ )
293
+
294
+
295
+ def sorting_from_times_labels(
296
+ times_samples,
297
+ labels,
298
+ recording=None,
299
+ motion_est=None,
300
+ sampling_frequency=30000.0,
301
+ determine_channels=True,
302
+ template_cfg=unshifted_raw_template_cfg,
303
+ n_jobs=0,
304
+ spikes_per_unit=50,
305
+ ):
306
+ channels = np.zeros_like(labels)
307
+ if sampling_frequency is None:
308
+ if recording is not None:
309
+ sampling_frequency = recording.sampling_frequency
310
+
311
+ efeat = {}
312
+ if recording is not None:
313
+ efeat["times_seconds"] = recording.sample_index_to_time(times_samples)
314
+
315
+ sorting = DARTsortSorting(
316
+ times_samples=times_samples,
317
+ channels=channels,
318
+ labels=labels,
319
+ sampling_frequency=sampling_frequency,
320
+ parent_h5_path=None,
321
+ ephemeral_features=efeat,
322
+ )
323
+
324
+ if not determine_channels:
325
+ return sorting
326
+
327
+ assert recording is not None
328
+
329
+ _, labels_flat = np.unique(labels, return_inverse=True)
330
+ sorting = sorting.ephemeral_replace(labels=labels_flat)
331
+ template_cfg = dataclasses.replace(template_cfg, spikes_per_unit=spikes_per_unit)
332
+ comp_cfg = ComputationConfig(n_jobs_cpu=n_jobs, n_jobs_gpu=n_jobs)
333
+ td = TemplateData.from_config(
334
+ recording, sorting, template_cfg, computation_cfg=comp_cfg
335
+ )
336
+
337
+ channels = np.nan_to_num(np.ptp(td.coarsen().templates, 1)).argmax(1)[labels_flat]
338
+ if motion_est is not None:
339
+ from scipy.spatial import KDTree
340
+
341
+ rgeom = td.registered_geom
342
+ assert rgeom is not None
343
+ guess_pos = rgeom[channels]
344
+ times_seconds = recording.sample_index_to_time(times_samples)
345
+ # anti-correct these already stable positions so that they start movin
346
+ guess_pos[:, 1] += motion_est.disp_at_s(times_seconds, depth_um=guess_pos[:, 1])
347
+
348
+ gkdt = KDTree(recording.get_channel_locations())
349
+ # closest original channels to shifted positions
350
+ # these positions can drift off the probe if the main channel does! so in that case
351
+ # we can't really upper bound the distance query. i guess it would be, like, the
352
+ # largest distance that a unit would ever extend, or something, but let's not worry.
353
+ d, channels = gkdt.query(guess_pos, workers=n_jobs)
354
+
355
+ assert isinstance(channels, np.ndarray)
356
+ sorting = sorting.ephemeral_replace(channels=channels)
357
+
358
+ return sorting, td
359
+
360
+
361
+ def sorting_from_spikeinterface(
362
+ sorting,
363
+ recording=None,
364
+ determine_channels=True,
365
+ template_cfg=unshifted_raw_template_cfg,
366
+ n_jobs=0,
367
+ ):
368
+ sv = sorting.to_spike_vector()
369
+ return sorting_from_times_labels(
370
+ sv["sample_index"],
371
+ sv["unit_index"],
372
+ sampling_frequency=sorting.sampling_frequency,
373
+ recording=recording,
374
+ determine_channels=determine_channels,
375
+ template_cfg=template_cfg,
376
+ n_jobs=n_jobs,
377
+ )
378
+
379
+
380
+ def _same(x):
381
+ return x
382
+
383
+
384
+ def load_dartsort_step_sortings(
385
+ sorting_dir,
386
+ load_simple_features=False,
387
+ load_feature_names=("times_seconds", "geom", "channel_index"),
388
+ detection_h5_names=(
389
+ "subtraction.h5",
390
+ "threshold.h5",
391
+ "universal.h5",
392
+ "matching0.h5",
393
+ ),
394
+ detection_h5_path: Path | str | None = None,
395
+ step_format="refined{step}",
396
+ recluster_format="recluster{step}",
397
+ mtime_gap_minutes=0,
398
+ name_formatter=None,
399
+ ) -> Generator[tuple[str, DARTsortSorting] | tuple[None, None], None, None]:
400
+ """Returns list of step names and sortings, ordered.
401
+
402
+ The mtime thing is trying to prevent reading hdf5 files which are in active
403
+ use, although its not a guarantee... h5 locking... need to figure it out.
404
+ """
405
+ mtime_dt = mtime_gap_minutes * 60 if mtime_gap_minutes else 0
406
+ if detection_h5_path is None:
407
+ for dh5n in detection_h5_names:
408
+ detection_h5_path = cast(Path, sorting_dir / dh5n)
409
+ if not detection_h5_path.exists():
410
+ continue
411
+ if mtime_dt:
412
+ age = time.time() - detection_h5_path.stat().st_mtime
413
+ if age < mtime_dt:
414
+ continue
415
+ h5s = [resolve_path(detection_h5_path)]
416
+ break
417
+ else:
418
+ h5s = []
419
+ else:
420
+ h5s = [resolve_path(detection_h5_path)]
421
+
422
+ for j in range(1, 100):
423
+ mh5 = sorting_dir / f"matching{j}.h5"
424
+ if not mh5.exists():
425
+ break
426
+ if mtime_dt:
427
+ if time.time() - mh5.stat().st_mtime < mtime_dt:
428
+ break
429
+ h5s.append(resolve_path(mh5))
430
+
431
+ # let's check that there is at least something to do...
432
+ labels_npys = sorting_dir.glob("*_labels.npy")
433
+ relevant_files = [sorting_dir / "dartsort_sorting.npz", *labels_npys, *h5s[1:]]
434
+ if not any(f.exists() for f in relevant_files):
435
+ h5s = []
436
+
437
+ if name_formatter is None:
438
+ name_formatter = _same
439
+
440
+ for step, h5 in enumerate(h5s):
441
+ if not h5.exists():
442
+ continue
443
+ st0 = DARTsortSorting.from_peeling_hdf5(
444
+ h5,
445
+ load_simple_features=load_simple_features,
446
+ load_feature_names=load_feature_names,
447
+ )
448
+
449
+ # initial clust or later step res?
450
+ if h5 == h5s[0]:
451
+ npy = sorting_dir / "initial_labels.npy"
452
+ if npy.exists():
453
+ yield (
454
+ name_formatter("initial"),
455
+ st0.ephemeral_replace(labels=np.load(npy)),
456
+ )
457
+ else:
458
+ warnings.warn(f"Initial {npy} does not exist.")
459
+ yield None, None
460
+
461
+ other_initial_npys = sorted(sorting_dir.glob("initial_*_labels.npy"))
462
+ for npy in other_initial_npys:
463
+ npy = sorting_dir / npy
464
+ stem = npy.stem.removesuffix("_labels")
465
+ yield (
466
+ name_formatter(stem),
467
+ st0.ephemeral_replace(labels=np.load(npy)),
468
+ )
469
+ else:
470
+ yield (name_formatter(h5.stem), st0)
471
+
472
+ # reclustering, if applicable
473
+ reclustr = recluster_format.format(step=step)
474
+ recluster_npy = sorting_dir / f"{reclustr}_labels.npy"
475
+ if recluster_npy.exists():
476
+ yield (
477
+ name_formatter(reclustr),
478
+ st0.ephemeral_replace(labels=np.load(recluster_npy)),
479
+ )
480
+
481
+ # refinement steps
482
+ stepstr = step_format.format(step=step)
483
+ for npy in sorted(sorting_dir.glob(f"{stepstr}*.npy")):
484
+ stem = npy.stem.removesuffix("_labels")
485
+ if stem == stepstr:
486
+ continue
487
+ yield (
488
+ name_formatter(stem),
489
+ st0.ephemeral_replace(labels=np.load(npy)),
490
+ )
491
+
492
+
493
+ def load_dartsort_step_unit_info_dataframes(
494
+ sorting_dir,
495
+ gt_analysis,
496
+ recording,
497
+ sorting_name=None,
498
+ detection_h5_names=(
499
+ "subtraction.h5",
500
+ "threshold.h5",
501
+ "universal.h5",
502
+ "matching0.h5",
503
+ ),
504
+ detection_h5_path: Path | str | None = None,
505
+ step_format="refined{step}",
506
+ ):
507
+ step_sortings = load_dartsort_step_sortings(
508
+ sorting_dir,
509
+ detection_h5_names=detection_h5_names,
510
+ detection_h5_path=detection_h5_path,
511
+ step_format=step_format,
512
+ )
513
+ for step_ix, (step_name, step_sorting) in enumerate(step_sortings):
514
+ if step_name is None:
515
+ continue
516
+ assert step_sorting is not None
517
+ name = f"{sorting_name}: {step_name}" if sorting_name else step_name
518
+ step_analysis = analysis.DARTsortAnalysis.from_sorting(
519
+ recording=recording,
520
+ sorting=step_sorting,
521
+ motion_est=None,
522
+ name=name,
523
+ template_cfg=None,
524
+ vis_radius=0,
525
+ )
526
+ step_comparison = comparison.DARTsortGroundTruthComparison(
527
+ gt_analysis, step_analysis
528
+ )
529
+ df = step_comparison.unit_info_dataframe()
530
+ df["stepix"] = step_ix
531
+ df["stepname"] = step_name
532
+ yield df
533
+
534
+
535
+ def config_grid(
536
+ common_params: dict | None = None,
537
+ name_prefix="",
538
+ config_cls: Any = DeveloperConfig,
539
+ **grid_params,
540
+ ):
541
+ """Configuration grid search helper
542
+
543
+ Generate a set product of aptly named configuration dictionaries or objects.
544
+
545
+ Example usage:
546
+
547
+ ```python
548
+ config_grid(
549
+ {'save_intermediate_labels': True, 'n_jobs_cpu': 2},
550
+ badness={".1":dict(denoiser_badness_factor=0.1),".25":dict(denoiser_badness_factor=0.25)},
551
+ crit={"hell":dict(merge_criterion="heldout_loglik"),"helb":dict(merge_criterion="heldout_elbo")},
552
+ )
553
+ => nice nested grid with names
554
+ ```
555
+
556
+ Arguments
557
+ ---------
558
+ common_params: dict
559
+ Arguments shared across all settings.
560
+ name_prefix: str
561
+ Will appear, with _, at the start of all keys in the returned dict.
562
+ config_cls: class or None
563
+ **grid_params
564
+ Each key-value pair here should be as follows. The key is like the
565
+ "tag" for this setting. The value is a dict whose keys are setting
566
+ nicknames and values are themselves dicts containing the actual
567
+ configurations. See the example above.
568
+
569
+ Returns
570
+ -------
571
+ grid: dict[str, config_cls]
572
+ My length is product(v for k, v in grid_params.items()).
573
+ """
574
+ if common_params is None:
575
+ common_params = {}
576
+
577
+ # base case (won't actually end up in the final grid)
578
+ grid = [(name_prefix, common_params)]
579
+
580
+ for param_nickname, param_settings in grid_params.items():
581
+ new_grid = []
582
+
583
+ for setting_nickname, kw in param_settings.items():
584
+ # induction
585
+ for cur_name, cur_params in grid:
586
+ prefix = f"{cur_name}_" if cur_name else ""
587
+ new_name = f"{prefix}{param_nickname}{setting_nickname}"
588
+ # new overrides old if overlapping
589
+ new_grid.append((new_name, cur_params | kw))
590
+
591
+ grid = new_grid
592
+
593
+ # convert to dict of configs
594
+ if config_cls is not None:
595
+ grid = {name: config_cls(**kw) for name, kw in grid}
596
+ else:
597
+ grid = dict(grid)
598
+
599
+ return grid