DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,528 @@
1
+ import gc
2
+ from dataclasses import dataclass, replace
3
+ from pathlib import Path
4
+ from sys import getrefcount
5
+
6
+ import numpy as np
7
+ import torch
8
+ from spikeinterface.core import BaseRecording
9
+ from tqdm.auto import trange
10
+ from sklearn.decomposition import TruncatedSVD, PCA
11
+
12
+ from ..localize.localize_util import localize_waveforms
13
+ from ..util import data_util, drift_util, job_util
14
+ from ..util.data_util import DARTsortSorting
15
+ from ..util.internal_config import (
16
+ ComputationConfig,
17
+ TemplateConfig,
18
+ WaveformConfig,
19
+ default_waveform_cfg,
20
+ )
21
+ from ..util.logging_util import get_logger
22
+ from ..util.spiketorch import fast_nanmedian, nanmean
23
+ from .get_templates import fit_tsvd, get_templates
24
+ from .superres_util import superres_sorting
25
+ from .template_util import weighted_average
26
+
27
+
28
+ logger = get_logger(__name__)
29
+
30
+
31
+ @dataclass
32
+ class TemplateData:
33
+ # (n_templates, spike_length_samples, n_registered_channels or n_channels)
34
+ templates: np.ndarray
35
+ # (n_templates,) maps template index to unit index (multiple templates can share a unit index)
36
+ unit_ids: np.ndarray
37
+ # (n_templates,) spike count for each template
38
+ spike_counts: np.ndarray
39
+ # (n_templates, n_registered_channels or n_channels) spike count for each channel
40
+ spike_counts_by_channel: np.ndarray | None = None
41
+ # (n_templates, spike_length_samples, n_registered_channels or n_channels)
42
+ raw_std_dev: np.ndarray | None = None
43
+
44
+ registered_geom: np.ndarray | None = None
45
+ trough_offset_samples: int = 42
46
+
47
+ # stores (n_templates, *) arrays of template properties
48
+ # possibilities:
49
+ # - motion_estimate_bin_centers
50
+ # For motion_estimate binned superres sortings, these are the drift
51
+ # bin centers for each template.
52
+ properties: dict[str, np.ndarray] | None = None
53
+ tsvd: TruncatedSVD | PCA | None = None
54
+
55
+ def __post_init__(self):
56
+ assert self.trough_offset_samples < self.spike_length_samples
57
+
58
+ ntemp = len(self.templates)
59
+ assert self.unit_ids.shape == (ntemp,)
60
+ assert self.spike_counts.shape == (ntemp,)
61
+ if self.spike_counts_by_channel is not None:
62
+ assert self.spike_counts_by_channel.ndim == 2
63
+ assert self.spike_counts_by_channel.shape[0] == ntemp
64
+ if self.raw_std_dev is not None:
65
+ assert self.raw_std_dev.shape == self.templates.shape
66
+
67
+ nc = self.templates.shape[2]
68
+ if self.spike_counts_by_channel is not None:
69
+ assert self.spike_counts_by_channel.shape[1] == nc
70
+ if self.registered_geom is not None:
71
+ assert self.registered_geom.ndim == 2
72
+ assert self.registered_geom.shape[0] == nc
73
+ if self.properties:
74
+ for v in self.properties.values():
75
+ assert v.shape[0] == ntemp
76
+
77
+ @property
78
+ def spike_length_samples(self):
79
+ return self.templates.shape[1]
80
+
81
+ def snrs_by_channel(self):
82
+ amp_vecs = np.nan_to_num(np.ptp(self.templates, axis=1), nan=-np.inf)
83
+ if self.spike_counts_by_channel is not None:
84
+ amp_vecs *= np.sqrt(self.spike_counts_by_channel)
85
+ return amp_vecs
86
+
87
+ def main_channels(self):
88
+ return self.snrs_by_channel().argmax(axis=1)
89
+
90
+ def template_locations(self, mode="channel", radius=100.0):
91
+ assert mode in ("localization", "channel")
92
+
93
+ if mode == "channel":
94
+ assert self.registered_geom is not None
95
+ return self.registered_geom[self.main_channels()]
96
+
97
+ assert mode == "localization"
98
+ rdepths = localize_waveforms(
99
+ waveforms=self.templates,
100
+ geom=self.registered_geom,
101
+ main_channels=self.main_channels(),
102
+ radius=radius,
103
+ )
104
+ rdepths = np.c_[rdepths["x"], rdepths["z_abs"]]
105
+ return rdepths
106
+
107
+ def registered_depths_um(self, mode="channel", radius=100.0):
108
+ return self.template_locations(mode=mode, radius=radius)[:, 1]
109
+
110
+ @classmethod
111
+ def from_npz(cls, npz_path):
112
+ with np.load(npz_path, allow_pickle=True) as data:
113
+ data = dict(**data)
114
+ if "spike_length_samples" in data:
115
+ del data["spike_length_samples"] # todo: remove
116
+ if "parent_sorting_hdf5_path" in data:
117
+ del data["parent_sorting_hdf5_path"] # todo: remove
118
+
119
+ properties = {}
120
+ for k, v in data.items():
121
+ if k.startswith("__prop_"):
122
+ properties[k.removeprefix("__prop_")] = v
123
+ for k in properties:
124
+ del data[f"__prop_{k}"]
125
+ if not properties:
126
+ properties = None
127
+
128
+ return cls(**data, properties=properties)
129
+
130
+ def to_npz(self, npz_path):
131
+ to_save = dict(
132
+ templates=self.templates,
133
+ unit_ids=self.unit_ids,
134
+ spike_counts=self.spike_counts,
135
+ trough_offset_samples=self.trough_offset_samples,
136
+ )
137
+ if self.registered_geom is not None:
138
+ to_save["registered_geom"] = self.registered_geom
139
+ if self.spike_counts_by_channel is not None:
140
+ to_save["spike_counts_by_channel"] = self.spike_counts_by_channel
141
+ if self.raw_std_dev is not None:
142
+ to_save["raw_std_dev"] = self.raw_std_dev
143
+ if not npz_path.parent.exists():
144
+ npz_path.parent.mkdir()
145
+ if self.properties is not None:
146
+ for k, p in self.properties.items():
147
+ to_save[f"__prop_{k}"] = p
148
+ np.savez(npz_path, **to_save) # type: ignore
149
+
150
+ def __getitem__(self, subset):
151
+ if not np.array_equal(self.unit_ids, np.arange(len(self.unit_ids))):
152
+ subset_ixs = np.searchsorted(self.unit_ids, subset, side="right") - 1
153
+ matched = self.unit_ids[subset_ixs] == subset
154
+ assert matched.all()
155
+ subset = subset_ixs
156
+ if self.properties is not None:
157
+ properties = {k: p[subset] for k, p in self.properties.items()}
158
+ else:
159
+ properties = None
160
+ return self.__class__(
161
+ templates=self.templates[subset],
162
+ unit_ids=self.unit_ids[subset],
163
+ spike_counts=self.spike_counts[subset],
164
+ spike_counts_by_channel=(
165
+ self.spike_counts_by_channel[subset]
166
+ if self.spike_counts_by_channel is not None
167
+ else None
168
+ ),
169
+ raw_std_dev=(
170
+ self.raw_std_dev[subset] if self.raw_std_dev is not None else None
171
+ ),
172
+ registered_geom=self.registered_geom,
173
+ trough_offset_samples=self.trough_offset_samples,
174
+ properties=properties,
175
+ tsvd=self.tsvd,
176
+ )
177
+
178
+ def coarsen(self):
179
+ """Weighted average all templates that share a unit id."""
180
+ # update templates
181
+ unit_ids_unique, flat_ids = np.unique(self.unit_ids, return_inverse=True)
182
+ templates = weighted_average(flat_ids, self.templates, self.spike_counts)
183
+
184
+ # collect spike counts
185
+ spike_counts = np.zeros(len(templates))
186
+ np.add.at(spike_counts, flat_ids, self.spike_counts)
187
+
188
+ return replace(
189
+ self,
190
+ templates=templates,
191
+ unit_ids=unit_ids_unique,
192
+ spike_counts=spike_counts,
193
+ properties=None,
194
+ tsvd=self.tsvd,
195
+ )
196
+
197
+ def unit_mask(self, unit_id):
198
+ return np.isin(self.unit_ids, unit_id)
199
+
200
+ def unit_templates(self, unit_id):
201
+ return self.templates[self.unit_mask(unit_id)]
202
+
203
+ @classmethod
204
+ def from_config(
205
+ cls,
206
+ recording: BaseRecording,
207
+ sorting: DARTsortSorting,
208
+ template_cfg: TemplateConfig,
209
+ waveform_cfg: WaveformConfig = default_waveform_cfg,
210
+ save_folder: Path | None = None,
211
+ overwrite=False,
212
+ motion_est=None,
213
+ save_npz_name: str | None = "template_data.npz",
214
+ units_per_job=8,
215
+ tsvd=None,
216
+ computation_cfg: ComputationConfig | None = None,
217
+ ):
218
+ self = _template_data_from_config(
219
+ cls,
220
+ recording=recording,
221
+ sorting=sorting,
222
+ template_cfg=template_cfg,
223
+ waveform_cfg=waveform_cfg,
224
+ save_folder=save_folder,
225
+ overwrite=overwrite,
226
+ motion_est=motion_est,
227
+ save_npz_name=save_npz_name,
228
+ units_per_job=units_per_job,
229
+ tsvd=tsvd,
230
+ computation_cfg=computation_cfg,
231
+ )
232
+ gc.collect()
233
+ torch.cuda.empty_cache()
234
+ return self
235
+
236
+
237
+ def _template_data_from_config(
238
+ cls,
239
+ recording: BaseRecording,
240
+ sorting: DARTsortSorting,
241
+ template_cfg: TemplateConfig,
242
+ waveform_cfg: WaveformConfig = default_waveform_cfg,
243
+ save_folder: str | Path | None = None,
244
+ overwrite: bool = False,
245
+ motion_est=None,
246
+ save_npz_name: str | None = "template_data.npz",
247
+ units_per_job=8,
248
+ tsvd=None,
249
+ computation_cfg: ComputationConfig | None = None,
250
+ show_progress: bool = True,
251
+ ) -> TemplateData:
252
+ if computation_cfg is None:
253
+ computation_cfg = job_util.get_global_computation_config()
254
+
255
+ npz_path = None
256
+ if save_folder is not None:
257
+ save_folder = Path(save_folder)
258
+ if not save_folder.exists():
259
+ save_folder.mkdir()
260
+ assert save_npz_name is not None
261
+ npz_path = save_folder / save_npz_name
262
+ if npz_path.exists() and not overwrite:
263
+ return cls.from_npz(npz_path)
264
+
265
+ if template_cfg.actual_algorithm() == "by_chunk":
266
+ template_data = get_templates_by_chunk(
267
+ sorting=sorting,
268
+ recording=recording,
269
+ tsvd=tsvd,
270
+ motion_est=motion_est,
271
+ waveform_cfg=waveform_cfg,
272
+ computation_cfg=computation_cfg,
273
+ template_cfg=template_cfg,
274
+ show_progress=show_progress,
275
+ )
276
+ return template_data
277
+
278
+ if sorting is None:
279
+ raise ValueError(
280
+ "TemplateData.from_config needs a sorting when its .npz file "
281
+ "does not exist."
282
+ )
283
+
284
+ fs = recording.sampling_frequency
285
+ trough_offset_samples = waveform_cfg.trough_offset_samples(fs)
286
+ spike_length_samples = waveform_cfg.spike_length_samples(fs)
287
+
288
+ if template_cfg.denoising_method in (None, "none"):
289
+ low_rank_denoising = False
290
+ else:
291
+ assert template_cfg.denoising_method == "exp_weighted"
292
+ low_rank_denoising = True
293
+
294
+ # load motion features if necessary
295
+ geom = recording.get_channel_locations()
296
+ if template_cfg.registered_templates:
297
+ motion_kw = dict(
298
+ motion_est=motion_est,
299
+ geom=geom,
300
+ localizations_dataset_name=template_cfg.localizations_dataset_name,
301
+ )
302
+ else:
303
+ motion_kw = dict(geom=geom)
304
+
305
+ # handle superresolved templates
306
+ if template_cfg.superres_templates:
307
+ superres_data = superres_sorting(
308
+ sorting=sorting,
309
+ geom=geom,
310
+ motion_est=motion_est,
311
+ strategy=template_cfg.superres_strategy,
312
+ superres_bin_size_um=template_cfg.superres_bin_size_um,
313
+ min_spikes_per_bin=template_cfg.superres_bin_min_spikes,
314
+ )
315
+ group_ids = superres_data["group_ids"]
316
+ assert isinstance(superres_data["sorting"], DARTsortSorting)
317
+ sorting = superres_data["sorting"]
318
+ properties = superres_data["properties"]
319
+ else:
320
+ group_ids = None
321
+ properties = {}
322
+
323
+ # main!
324
+ results = get_templates(
325
+ recording=recording,
326
+ sorting=sorting,
327
+ trough_offset_samples=trough_offset_samples,
328
+ spike_length_samples=spike_length_samples,
329
+ spikes_per_unit=template_cfg.spikes_per_unit,
330
+ denoising_rank=template_cfg.denoising_rank,
331
+ recompute_tsvd=template_cfg.recompute_tsvd,
332
+ denoising_fit_radius=template_cfg.denoising_fit_radius,
333
+ denoising_snr_threshold=template_cfg.exp_weight_snr_threshold,
334
+ units_per_job=units_per_job,
335
+ with_raw_std_dev=template_cfg.with_raw_std_dev,
336
+ reducer=nanmean if template_cfg.reduction == "mean" else fast_nanmedian,
337
+ low_rank_denoising=low_rank_denoising,
338
+ denoising_tsvd=tsvd,
339
+ device=computation_cfg.actual_device(),
340
+ n_jobs=computation_cfg.actual_n_jobs(),
341
+ show_progress=show_progress,
342
+ **motion_kw, # type: ignore
343
+ )
344
+ if template_cfg.superres_templates:
345
+ assert group_ids is not None
346
+ unit_ids = group_ids[results["unit_ids"]] # type: ignore
347
+ else:
348
+ unit_ids = results["unit_ids"]
349
+ rgeom = results["registered_geom"]
350
+ if rgeom is None:
351
+ rgeom = geom
352
+ if tsvd is None:
353
+ tsvd = results["denoising_tsvd"]
354
+ obj = cls(
355
+ results["templates"],
356
+ unit_ids=unit_ids,
357
+ spike_counts=results["spike_counts"],
358
+ spike_counts_by_channel=results["spike_counts_by_channel"],
359
+ raw_std_dev=results["raw_std_devs"],
360
+ registered_geom=rgeom,
361
+ trough_offset_samples=trough_offset_samples,
362
+ properties=properties,
363
+ tsvd=tsvd,
364
+ )
365
+ if save_folder is not None:
366
+ obj.to_npz(npz_path)
367
+
368
+ return obj
369
+
370
+
371
+ def get_templates_by_chunk(
372
+ *,
373
+ sorting: DARTsortSorting,
374
+ recording: BaseRecording,
375
+ tsvd,
376
+ motion_est,
377
+ waveform_cfg: WaveformConfig,
378
+ computation_cfg: ComputationConfig,
379
+ template_cfg: TemplateConfig,
380
+ show_progress: bool,
381
+ block_size=384,
382
+ hard_block_size=512,
383
+ random_seed=0,
384
+ dtype=np.float32,
385
+ ) -> TemplateData:
386
+ if template_cfg.denoising_method == "none":
387
+ block_size = hard_block_size = template_cfg.raw_templates_at_once
388
+ else:
389
+ block_size = template_cfg.templates_at_once
390
+ hard_block_size = template_cfg.max_templates_at_once
391
+
392
+ assert sorting.labels is not None
393
+ unit_ids = np.unique(sorting.labels)
394
+ unit_ids = unit_ids[unit_ids >= 0]
395
+ n_units = unit_ids.shape[0]
396
+ n_blocks = max(1, n_units // block_size)
397
+ n_blocks += n_units / n_blocks > hard_block_size
398
+ assert n_units / n_blocks <= hard_block_size
399
+ units_per_block = int(np.ceil(n_units / n_blocks).astype(int).item())
400
+
401
+ need_tsvd = template_cfg.denoising_method != "none"
402
+ if need_tsvd and tsvd is None:
403
+ logger.dartsortdebug("Fit or load TSVD...")
404
+ tsvd = fit_tsvd(
405
+ recording,
406
+ sorting,
407
+ dtype=dtype,
408
+ denoising_rank=template_cfg.denoising_rank,
409
+ denoising_fit_radius=template_cfg.denoising_fit_radius,
410
+ denoising_spikes_fit=template_cfg.denoising_fit_sampling_cfg.n_waveforms_fit,
411
+ recompute_tsvd=template_cfg.recompute_tsvd,
412
+ trough_offset_samples=waveform_cfg.trough_offset_samples(
413
+ recording.sampling_frequency
414
+ ),
415
+ spike_length_samples=waveform_cfg.spike_length_samples(
416
+ recording.sampling_frequency
417
+ ),
418
+ random_seed=random_seed,
419
+ )
420
+
421
+ labels_tmp = np.full_like(sorting.labels, -1)
422
+ template_datas = []
423
+ if n_blocks > 1:
424
+ block_iter = trange(
425
+ 0, n_units, units_per_block, desc=f"Template blocks [{n_units}:{n_blocks}]"
426
+ )
427
+ else:
428
+ block_iter = range(0, n_units, units_per_block)
429
+ for block_start in block_iter:
430
+ block_end = min(block_start + units_per_block, n_units)
431
+ if n_blocks == 1:
432
+ assert block_end - block_start == n_units
433
+
434
+ ids_in_block = unit_ids[block_start:block_end]
435
+ spikes_in_block = np.flatnonzero(np.isin(sorting.labels, ids_in_block))
436
+ labels_tmp[:] = -1
437
+ labels_tmp[spikes_in_block] = sorting.labels[spikes_in_block]
438
+
439
+ block_sorting = sorting.ephemeral_replace(labels=labels_tmp)
440
+ block_templates = _get_templates_by_chunk_block(
441
+ sorting=block_sorting,
442
+ recording=recording,
443
+ tsvd=tsvd,
444
+ motion_est=motion_est,
445
+ waveform_cfg=waveform_cfg,
446
+ computation_cfg=computation_cfg,
447
+ template_cfg=template_cfg,
448
+ show_progress=show_progress,
449
+ )
450
+ template_datas.append(block_templates)
451
+ assert np.array_equal(block_templates.unit_ids, ids_in_block)
452
+
453
+ template_data = stack_template_datas(template_datas)
454
+ assert np.array_equal(template_data.unit_ids, unit_ids)
455
+ assert template_data.templates.shape[0] == unit_ids.shape[0]
456
+ return template_data
457
+
458
+
459
+ def _get_templates_by_chunk_block(
460
+ sorting: DARTsortSorting,
461
+ recording,
462
+ tsvd,
463
+ motion_est,
464
+ waveform_cfg,
465
+ computation_cfg,
466
+ template_cfg,
467
+ show_progress: bool,
468
+ ):
469
+ from ..peel.running_template import RunningTemplates
470
+
471
+ peeler = RunningTemplates.from_config(
472
+ sorting=sorting,
473
+ recording=recording,
474
+ tsvd=tsvd,
475
+ motion_est=motion_est,
476
+ waveform_cfg=waveform_cfg,
477
+ template_cfg=template_cfg,
478
+ )
479
+
480
+ template_data = peeler.compute_template_data(
481
+ show_progress=show_progress, computation_cfg=computation_cfg
482
+ )
483
+
484
+ assert getrefcount(peeler) == 2, (
485
+ f"Leaking the template peeler {getrefcount(peeler)=}."
486
+ )
487
+ del peeler
488
+ gc.collect()
489
+ torch.cuda.empty_cache()
490
+
491
+ return template_data
492
+
493
+
494
+ def stack_template_datas(template_datas):
495
+ if len(template_datas) == 1:
496
+ return template_datas[0]
497
+ assert len(template_datas) > 0
498
+
499
+ if template_datas[0].spike_counts_by_channel is not None:
500
+ spike_counts_by_channel = np.concatenate(
501
+ [td.spike_counts_by_channel for td in template_datas]
502
+ )
503
+ else:
504
+ spike_counts_by_channel = None
505
+
506
+ if template_datas[0].raw_std_dev is not None:
507
+ raw_std_dev = np.concatenate([td.raw_std_dev for td in template_datas])
508
+ else:
509
+ raw_std_dev = None
510
+
511
+ if template_datas[0].properties is not None:
512
+ properties = {
513
+ k: np.concatenate([td.properties[k] for td in template_datas])
514
+ for k in template_datas[0].properties
515
+ }
516
+ else:
517
+ properties = None
518
+
519
+ return TemplateData(
520
+ templates=np.concatenate([td.templates for td in template_datas]),
521
+ unit_ids=np.concatenate([td.unit_ids for td in template_datas]),
522
+ spike_counts=np.concatenate([td.spike_counts for td in template_datas]),
523
+ spike_counts_by_channel=spike_counts_by_channel,
524
+ raw_std_dev=raw_std_dev,
525
+ registered_geom=template_datas[0].registered_geom,
526
+ trough_offset_samples=template_datas[0].trough_offset_samples,
527
+ properties=properties,
528
+ )
@@ -0,0 +1,2 @@
1
+ from .all_transformers import *
2
+ from .pipeline import WaveformPipeline