DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1000 @@
1
+ import gc
2
+ import tempfile
3
+ from concurrent.futures import CancelledError
4
+ from contextlib import contextmanager
5
+ from itertools import repeat
6
+ from pathlib import Path
7
+ from sys import getrefcount
8
+ from threading import Lock, local
9
+ from types import MappingProxyType
10
+ from typing import Any, Literal, TypedDict, overload
11
+
12
+ import h5py
13
+ import numpy as np
14
+ import torch
15
+ from spikeinterface.core.recording_tools import get_chunk_with_margin
16
+ from sympy import divisors
17
+ from tqdm.auto import tqdm
18
+
19
+ from ..transform import WaveformPipeline
20
+ from ..util import job_util
21
+ from ..util.data_util import (
22
+ SpikeDataset,
23
+ divide_randomly,
24
+ extract_random_snips,
25
+ subsample_waveforms,
26
+ )
27
+ from ..util.logging_util import get_logger
28
+ from ..util.multiprocessing_util import pool_from_cfg
29
+ from ..util.py_util import delay_keyboard_interrupt
30
+ from ..util.torch_util import BModule
31
+
32
+ logger = get_logger(__name__)
33
+ _lock = Lock()
34
+
35
+
36
+ class BasePeeler(BModule):
37
+ """Base class for peeling operations (subtraction, deconv, etc)
38
+
39
+ Subtraction, deconv, and other things like just grabbing waveforms
40
+ and featurizing a preexisting spike train (implemented with optional
41
+ additional peeling of preexisting templates in grab.py) share a lot
42
+ of logic for featurization and parallelization and data saving/loading.
43
+ This class handles all of that stuff so that it can be shared.
44
+ """
45
+
46
+ peel_kind = ""
47
+
48
+ def __init__(
49
+ self,
50
+ recording,
51
+ channel_index,
52
+ featurization_pipeline: WaveformPipeline | None = None,
53
+ chunk_length_samples=30_000,
54
+ chunk_margin_samples=0,
55
+ n_seconds_fit=40,
56
+ max_waveforms_fit=50_000,
57
+ n_waveforms_fit=20_000,
58
+ fit_max_reweighting=4.0,
59
+ fit_sampling: Literal["random", "amp_reweighted"] = "random",
60
+ fit_subsampling_random_state: int | np.random.Generator = 0,
61
+ trough_offset_samples=42,
62
+ spike_length_samples=121,
63
+ fixed_property_keys=("channels",),
64
+ dtype=torch.float,
65
+ ):
66
+ if recording.get_num_segments() > 1:
67
+ raise ValueError("Peeling does not yet support multi-segment recordings.")
68
+ super().__init__()
69
+ self.recording = recording
70
+ self.chunk_length_samples: int = chunk_length_samples
71
+ self.chunk_margin_samples: int = chunk_margin_samples
72
+ self.n_seconds_fit: int = n_seconds_fit
73
+ self.max_waveforms_fit: int = max_waveforms_fit
74
+ self.n_waveforms_fit: int = n_waveforms_fit
75
+ self.trough_offset_samples: int = trough_offset_samples
76
+ self.spike_length_samples: int = spike_length_samples
77
+ self.fit_subsampling_random_state: np.random.Generator = np.random.default_rng(
78
+ fit_subsampling_random_state
79
+ )
80
+ self.dtype: torch.dtype = dtype
81
+ self.np_dtype = torch.empty((), dtype=dtype).numpy().dtype
82
+ if channel_index is not None:
83
+ self.register_buffer("channel_index", channel_index)
84
+ assert recording.get_num_channels() == channel_index.shape[0]
85
+ self.fit_sampling: Literal["random", "amp_reweighted"] = fit_sampling
86
+ self.fit_max_reweighting: float = fit_max_reweighting
87
+ self.featurization_pipeline: WaveformPipeline | None = featurization_pipeline
88
+ self.fixed_property_keys: tuple[str] | list[str] = fixed_property_keys
89
+
90
+ # subclasses can append to this if they want to store more fixed
91
+ # arrays in the output h5 file
92
+ self.fixed_output_data: list[tuple[str, Any]] = [
93
+ ("sampling_frequency", self.recording.get_sampling_frequency()),
94
+ ("geom", self.recording.get_channel_locations()),
95
+ ]
96
+ if channel_index is not None:
97
+ self.fixed_output_data.append(
98
+ ("channel_index", self.b.channel_index.numpy(force=True).copy()),
99
+ )
100
+
101
+ self._rgs: local = local()
102
+
103
+ # -- main functions for users to call
104
+ # in practice users will interact with the functions `subtract(...)` in
105
+ # subtract.py and similar functions in the other .py files here, but these
106
+ # are the main API methods for this class
107
+
108
+ def load_or_fit_and_save_models(
109
+ self, save_folder, overwrite=False, computation_cfg=None
110
+ ):
111
+ """Load fitted models from save_folder if possible, or fit and save
112
+
113
+ If the peeler has models that need to be trained, this function ensures
114
+ that the models are fitted and that their fitted parameters are saved
115
+ to `save_folder`
116
+ """
117
+ save_folder = Path(save_folder)
118
+ if overwrite and save_folder.exists():
119
+ for pt_file in save_folder.glob("*pipeline.pt"):
120
+ pt_file.unlink()
121
+ if self.needs_precompute() or self.needs_fit():
122
+ self.load_models(save_folder)
123
+ if self.needs_precompute() or self.needs_fit():
124
+ if self.needs_precompute():
125
+ self.precompute_models(
126
+ save_folder, overwrite=overwrite, computation_cfg=computation_cfg
127
+ )
128
+ if self.needs_fit():
129
+ save_folder.mkdir(exist_ok=True)
130
+ self.fit_models(
131
+ save_folder, overwrite=overwrite, computation_cfg=computation_cfg
132
+ )
133
+ self.save_models(save_folder)
134
+ assert not self.needs_precompute()
135
+ assert not self.needs_fit()
136
+
137
+ def peel(
138
+ self,
139
+ output_hdf5_filename,
140
+ chunk_starts_samples=None,
141
+ chunk_length_samples=None,
142
+ total_residual_snips=None,
143
+ residual_snips_per_chunk=None,
144
+ stop_after_n_waveforms=None,
145
+ shuffle=False,
146
+ overwrite=False,
147
+ residual_filename=None,
148
+ show_progress=True,
149
+ skip_features=False,
150
+ residual_to_h5=False,
151
+ task_name=None,
152
+ ignore_resuming=False,
153
+ computation_cfg=None,
154
+ ):
155
+ """Run the full (already fitted) peeling and featurization pipeline
156
+
157
+ This gathers all results into an hdf5 file, resuming from where
158
+ it left off in that file if overwrite=False.
159
+ """
160
+ if self.needs_fit():
161
+ raise ValueError("Peeler needs to be fitted before peeling.")
162
+
163
+ if output_hdf5_filename is None:
164
+ # all of these features would require the h5 file.
165
+ assert ignore_resuming
166
+ assert not residual_to_h5
167
+ assert residual_snips_per_chunk is None
168
+ assert total_residual_snips is None
169
+
170
+ if task_name is None:
171
+ task_name = self.peel_kind
172
+
173
+ # this is -1 if we haven't started yet
174
+ if ignore_resuming:
175
+ last_chunk_start = -1
176
+ resids_so_far = 0
177
+ elif output_hdf5_filename is not None:
178
+ last_chunk_start, resids_so_far = self.check_resuming(
179
+ output_hdf5_filename, overwrite=overwrite
180
+ )
181
+ logger.dartsortdebug(
182
+ f"[{self.__class__.__name__}:{task_name}] Resuming at {last_chunk_start=}."
183
+ )
184
+ else:
185
+ assert False
186
+
187
+ # figure out which chunks to process, and exit early if already done
188
+ chunk_starts_samples = self.get_chunk_starts(
189
+ chunk_starts_samples=chunk_starts_samples,
190
+ )
191
+ chunk_starts_samples = np.asarray(chunk_starts_samples, dtype=int)
192
+ if shuffle:
193
+ assert overwrite or last_chunk_start < 0
194
+ self.fit_subsampling_random_state.shuffle(chunk_starts_samples)
195
+ n_chunks_orig = len(chunk_starts_samples)
196
+ chunks_to_do = [
197
+ start for start in chunk_starts_samples if start > last_chunk_start
198
+ ]
199
+
200
+ if total_residual_snips is not None:
201
+ assert residual_snips_per_chunk is None
202
+ resids_remaining = total_residual_snips - resids_so_far
203
+ residual_snips_per_chunk = divide_randomly(
204
+ resids_remaining, len(chunks_to_do), self.fit_subsampling_random_state
205
+ )
206
+
207
+ if residual_snips_per_chunk is None:
208
+ residual_snips_per_chunk = repeat(None)
209
+ elif isinstance(residual_snips_per_chunk, int):
210
+ residual_to_h5 = bool(residual_snips_per_chunk)
211
+ residual_snips_per_chunk = repeat(residual_snips_per_chunk)
212
+ else:
213
+ residual_to_h5 = True
214
+ assert len(residual_snips_per_chunk) == len(chunks_to_do)
215
+
216
+ jobs = list(zip(chunks_to_do, residual_snips_per_chunk))
217
+
218
+ if not chunks_to_do:
219
+ return output_hdf5_filename
220
+ save_residual = residual_filename is not None
221
+ compute_residual = save_residual or residual_to_h5
222
+ if chunk_length_samples is None:
223
+ chunk_length_samples = self.chunk_length_samples
224
+ if computation_cfg is None:
225
+ computation_cfg = job_util.get_global_computation_config()
226
+
227
+ to_cpu = output_hdf5_filename is not None
228
+
229
+ # main peeling loop
230
+ # wrap in try/finally to ensure file handles get closed if there
231
+ # is some unforseen error
232
+ try:
233
+ n_jobs, Executor, context, rank_queue, is_local = pool_from_cfg( # type: ignore
234
+ computation_cfg, with_rank_queue=True, check_local=True
235
+ )
236
+ with Executor(
237
+ max_workers=n_jobs,
238
+ mp_context=context,
239
+ initializer=_peeler_process_init,
240
+ initargs=(
241
+ self,
242
+ computation_cfg.actual_device(),
243
+ rank_queue,
244
+ compute_residual,
245
+ skip_features,
246
+ chunk_length_samples,
247
+ is_local,
248
+ to_cpu,
249
+ ),
250
+ ) as pool:
251
+ if is_local:
252
+ self.to(computation_cfg.actual_device())
253
+
254
+ # launch the jobs and wrap in a progress bar
255
+ results = pool.map(_peeler_process_job, jobs)
256
+ if show_progress:
257
+ n_sec_chunk = (
258
+ chunk_length_samples / self.recording.get_sampling_frequency()
259
+ )
260
+ dtag = computation_cfg.actual_device().type
261
+ results = tqdm(
262
+ results,
263
+ total=n_chunks_orig,
264
+ initial=n_chunks_orig - len(chunks_to_do),
265
+ smoothing=0,
266
+ desc=f"{task_name}:{dtag} {n_sec_chunk:.1f}s/it [spk/it=%%%]",
267
+ mininterval=0.25,
268
+ )
269
+ else:
270
+ n_sec_chunk = None
271
+
272
+ # construct h5 after forking to avoid pickling it
273
+ with self.initialize_files(
274
+ output_hdf5_filename,
275
+ residual_filename=residual_filename,
276
+ overwrite=overwrite,
277
+ skip_features=skip_features,
278
+ residual_to_h5=residual_to_h5,
279
+ ) as (
280
+ output_h5,
281
+ h5_spike_datasets,
282
+ residual_file,
283
+ n_spikes,
284
+ ):
285
+ batch_count = 0
286
+ try:
287
+ for result, chunk_start_samples in zip(results, chunks_to_do):
288
+ n_new_spikes = self.gather_chunk_result(
289
+ n_spikes,
290
+ chunk_start_samples,
291
+ result,
292
+ h5_spike_datasets,
293
+ output_h5,
294
+ residual_file,
295
+ residual_to_h5,
296
+ ignore_resuming,
297
+ skip_features,
298
+ )
299
+ batch_count += 1
300
+ n_spikes += n_new_spikes
301
+ if show_progress:
302
+ desc = f"{task_name}"
303
+ if not skip_features:
304
+ assert n_sec_chunk is not None
305
+ desc += f" [spk/{n_sec_chunk:g}s={n_spikes / batch_count:0.1f}]"
306
+ results.set_description(desc, refresh=False) # type: ignore
307
+ if not skip_features and (
308
+ stop_after_n_waveforms
309
+ and n_spikes >= stop_after_n_waveforms
310
+ ):
311
+ pool.shutdown(cancel_futures=True)
312
+ except CancelledError:
313
+ if show_progress:
314
+ results.write( # type: ignore
315
+ f"Got {n_spikes} spikes, enough to stop early."
316
+ )
317
+ finally:
318
+ self.to("cpu")
319
+
320
+ # we very much do not want to leak self into a global and have its
321
+ # memory tied up while dartsort wants to go do other stuff
322
+ global _peeler_process_context
323
+ del _peeler_process_context.ctx
324
+ _peeler_process_context.ctx = None
325
+
326
+ return output_hdf5_filename
327
+
328
+ # -- methods for subclasses to override
329
+
330
+ def peel_chunk(
331
+ self,
332
+ traces,
333
+ *,
334
+ chunk_start_samples=0,
335
+ left_margin=0,
336
+ right_margin=0,
337
+ return_residual=False,
338
+ return_waveforms=True,
339
+ ) -> "PeelingBatchResult":
340
+ # subclasses should implement this method
341
+
342
+ # they should return a dictionary with keys:
343
+ # - n_spikes
344
+ # - collisioncleaned_waveforms
345
+ # - times_samples (relative to start of traces)
346
+ # - channels
347
+ # - residual if requested
348
+ # - arbitrary subclass-specific stuff which should have keys in out_datasets()
349
+
350
+ # data for spikes in the margin should not be returned
351
+ # these spikes will have been processed by the neighboring chunk
352
+
353
+ raise NotImplementedError
354
+
355
+ def peeling_needs_fit(self) -> bool:
356
+ return False
357
+
358
+ def peeling_needs_precompute(self) -> bool:
359
+ return False
360
+
361
+ def precompute_peeling_data(
362
+ self, save_folder, overwrite=False, computation_cfg=None
363
+ ):
364
+ # subclasses should override if they need to cache data for peeling
365
+ # runs before fit_peeler_models()
366
+ pass
367
+
368
+ def fit_peeler_models(self, save_folder, tmp_dir=None, computation_cfg=None):
369
+ # subclasses should override if they need to fit models for peeling
370
+ assert not self.peeling_needs_fit()
371
+
372
+ def precompute_peeler_models(
373
+ self, save_folder, overwrite=False, computation_cfg=None
374
+ ):
375
+ if self.peeling_needs_precompute():
376
+ self.precompute_peeling_data(
377
+ save_folder=save_folder,
378
+ overwrite=overwrite,
379
+ computation_cfg=computation_cfg,
380
+ )
381
+ assert not self.peeling_needs_precompute()
382
+
383
+ # subclasses can add to this list
384
+ # for each dataset in this list, an output dataset in the
385
+ # hdf5 (of .peel()) will be created with this dtype and with
386
+ # shape (N_spikes, *shape_per_spike)
387
+ # datasets will also be created corresponding to features
388
+ # in featurization_pipeline
389
+ def out_datasets(self):
390
+ datasets = [
391
+ SpikeDataset(name="times_samples", shape_per_spike=(), dtype=np.int64),
392
+ SpikeDataset(name="times_seconds", shape_per_spike=(), dtype=np.float64),
393
+ SpikeDataset(name="channels", shape_per_spike=(), dtype=np.int64),
394
+ ]
395
+ if self.featurization_pipeline is not None:
396
+ datasets.extend(self.featurization_pipeline.spike_datasets())
397
+ return datasets
398
+
399
+ # -- utility methods which users likely won't touch
400
+
401
+ def featurize_collisioncleaned_waveforms(
402
+ self, collisioncleaned_waveforms, **fixed_properties
403
+ ):
404
+ if not self.featurization_pipeline:
405
+ return {}
406
+
407
+ waveforms, features = self.featurization_pipeline(
408
+ collisioncleaned_waveforms, **fixed_properties
409
+ )
410
+ return features
411
+
412
+ def process_chunk(
413
+ self,
414
+ chunk_start_samples: int,
415
+ *,
416
+ chunk_end_samples: int | None = None,
417
+ return_residual: bool = False,
418
+ skip_features: bool = False,
419
+ n_resid_snips: int | None = None,
420
+ to_cpu: bool = True,
421
+ ) -> "PeelingBatchResult":
422
+ """Grab, peel, and featurize a chunk, returning a dict of numpy arrays
423
+
424
+ Main function called in peeling workers
425
+ """
426
+ if chunk_end_samples is None:
427
+ chunk_end_samples = chunk_start_samples + self.chunk_length_samples
428
+ chunk_end_samples = min(self.recording.get_num_samples(), chunk_end_samples)
429
+ chunk, left_margin, right_margin = get_chunk_with_margin(
430
+ self.recording._recording_segments[0],
431
+ start_frame=chunk_start_samples,
432
+ end_frame=chunk_end_samples,
433
+ channel_indices=None,
434
+ margin=self.chunk_margin_samples,
435
+ )
436
+ chunk = torch.tensor(
437
+ chunk, device=self.b.channel_index.device, dtype=self.dtype
438
+ )
439
+ return_waveforms = not skip_features and bool(self.featurization_pipeline)
440
+ peel_result = self.peel_chunk(
441
+ chunk,
442
+ chunk_start_samples=chunk_start_samples,
443
+ left_margin=left_margin,
444
+ right_margin=right_margin,
445
+ return_waveforms=return_waveforms,
446
+ return_residual=return_residual or bool(n_resid_snips),
447
+ )
448
+
449
+ if peel_result["n_spikes"] > 0 and return_waveforms:
450
+ fixed_properties = {k: peel_result[k] for k in self.fixed_property_keys}
451
+ features = self.featurize_collisioncleaned_waveforms(
452
+ peel_result["collisioncleaned_waveforms"], **fixed_properties
453
+ )
454
+ else:
455
+ features = {}
456
+
457
+ # a user who wants these must featurize with a waveform node
458
+ # then they'll end up in `features`
459
+ if "collisioncleaned_waveforms" in peel_result:
460
+ del peel_result["collisioncleaned_waveforms"]
461
+
462
+ assert not any(k in features for k in peel_result)
463
+ chunk_result = {**peel_result, **features}
464
+ if to_cpu:
465
+ chunk_result = {
466
+ k: v.cpu() if torch.is_tensor(v) else v for k, v in chunk_result.items()
467
+ }
468
+
469
+ if "residual" in peel_result:
470
+ if torch.is_tensor(peel_result["residual"]):
471
+ peel_result["residual"] = peel_result["residual"].cpu() # type: ignore
472
+
473
+ # add times in seconds
474
+ segment = self.recording._recording_segments[0]
475
+ chunk_result["chunk_start_seconds"] = segment.sample_index_to_time(
476
+ chunk_start_samples
477
+ )
478
+ if peel_result["n_spikes"] and to_cpu:
479
+ chunk_result["times_seconds"] = segment.sample_index_to_time(
480
+ chunk_result["times_samples"]
481
+ )
482
+
483
+ if n_resid_snips:
484
+ chunk_result["resid_snips"], resid_times_samples = extract_random_snips(
485
+ self.rg,
486
+ peel_result["residual"],
487
+ n_resid_snips,
488
+ self.spike_length_samples,
489
+ )
490
+ chunk_result["residual_times_seconds"] = (
491
+ self.recording.sample_index_to_time(
492
+ chunk_start_samples + resid_times_samples
493
+ )
494
+ )
495
+
496
+ return chunk_result # type: ignore
497
+
498
+ def gather_chunk_result(
499
+ self,
500
+ cur_n_spikes,
501
+ chunk_start_samples,
502
+ chunk_result,
503
+ h5_spike_datasets,
504
+ output_h5,
505
+ residual_file,
506
+ residual_to_h5,
507
+ ignore_resuming,
508
+ skip_features,
509
+ ):
510
+ # delay keyboard interrupts so we don't write half a batch
511
+ # of data and leave files in an invalid state after ^C
512
+ # not that something else couldn't happen...
513
+ with delay_keyboard_interrupt:
514
+ if not ignore_resuming:
515
+ output_h5["last_chunk_start"][()] = chunk_start_samples
516
+
517
+ if residual_file is not None:
518
+ chunk_result["residual"].tofile(residual_file)
519
+
520
+ if "resid_snips" in chunk_result:
521
+ n_residuals = len(output_h5["residual"])
522
+ n_new_res = len(chunk_result["resid_snips"])
523
+ assert chunk_result["residual_times_seconds"].shape == (n_new_res,)
524
+ output_h5["residual"].resize(n_residuals + n_new_res, axis=0)
525
+ output_h5["residual"][n_residuals:] = chunk_result["resid_snips"]
526
+ output_h5["residual_times_seconds"].resize(
527
+ n_residuals + n_new_res, axis=0
528
+ )
529
+ output_h5["residual_times_seconds"][n_residuals:] = chunk_result[
530
+ "residual_times_seconds"
531
+ ]
532
+
533
+ if skip_features:
534
+ return 0
535
+
536
+ n_new_spikes = chunk_result["n_spikes"]
537
+ if not n_new_spikes:
538
+ return 0
539
+
540
+ for ds in self.out_datasets():
541
+ h5ds = h5_spike_datasets[ds.name]
542
+ h5ds.resize(cur_n_spikes + n_new_spikes, axis=0)
543
+ h5ds[cur_n_spikes:] = chunk_result[ds.name]
544
+
545
+ return n_new_spikes
546
+
547
+ def needs_fit(self):
548
+ if self.peeling_needs_fit():
549
+ return True
550
+ if self.featurization_pipeline is not None:
551
+ return self.featurization_pipeline.needs_fit()
552
+ return False
553
+
554
+ def needs_precompute(self):
555
+ if self.peeling_needs_precompute():
556
+ return True
557
+ if self.featurization_pipeline is not None:
558
+ return self.featurization_pipeline.needs_precompute()
559
+ return False
560
+
561
+ def fit_models(
562
+ self, save_folder, tmp_dir=None, overwrite=False, computation_cfg=None
563
+ ):
564
+ with torch.no_grad():
565
+ if self.peeling_needs_fit():
566
+ self.precompute_peeling_data(
567
+ save_folder=save_folder,
568
+ overwrite=overwrite,
569
+ computation_cfg=computation_cfg,
570
+ )
571
+ self.fit_peeler_models(
572
+ save_folder=save_folder,
573
+ tmp_dir=tmp_dir,
574
+ computation_cfg=computation_cfg,
575
+ )
576
+ self.fit_featurization_pipeline(
577
+ tmp_dir=tmp_dir,
578
+ computation_cfg=computation_cfg,
579
+ )
580
+ assert not self.needs_fit()
581
+
582
+ def precompute_models(self, save_folder, overwrite=False, computation_cfg=None):
583
+ if self.peeling_needs_precompute():
584
+ self.precompute_peeler_models(
585
+ save_folder=save_folder,
586
+ overwrite=overwrite,
587
+ computation_cfg=computation_cfg,
588
+ )
589
+ if self.featurization_pipeline is None:
590
+ return
591
+ self.featurization_pipeline.precompute()
592
+
593
+ def fit_featurization_pipeline(self, tmp_dir=None, computation_cfg=None):
594
+ if self.featurization_pipeline is None:
595
+ return
596
+
597
+ if not self.featurization_pipeline.needs_fit():
598
+ return
599
+
600
+ if computation_cfg is None:
601
+ computation_cfg = job_util.get_global_computation_config()
602
+ device = computation_cfg.actual_device()
603
+
604
+ # disable self's featurization pipeline, replacing it with a waveform
605
+ # saving node. then, we'll use those waveforms to fit the original
606
+ featurization_pipeline = self.featurization_pipeline
607
+ self.featurization_pipeline = WaveformPipeline.from_class_names_and_kwargs(
608
+ self.recording.get_channel_locations(),
609
+ self.channel_index,
610
+ [
611
+ ("Voltage", {"name": "peeled_voltages_fit"}),
612
+ (
613
+ "Waveform",
614
+ {
615
+ "name": "peeled_waveforms_fit",
616
+ "spike_length_samples": self.spike_length_samples,
617
+ },
618
+ ),
619
+ ],
620
+ )
621
+
622
+ with tempfile.TemporaryDirectory(dir=tmp_dir) as temp_dir:
623
+ temp_hdf5_filename = Path(temp_dir) / "peeler_fit.h5"
624
+ waveforms = fixed_properties = None
625
+ try:
626
+ self.run_subsampled_peeling(
627
+ temp_hdf5_filename,
628
+ computation_cfg=computation_cfg,
629
+ task_name="Load examples for feature fitting",
630
+ )
631
+
632
+ # fit featurization pipeline and reassign
633
+ # work in a try finally so we can delete the temp file
634
+ # in case of an issue or a keyboard interrupt
635
+ waveforms, fixed_properties = subsample_waveforms(
636
+ temp_hdf5_filename,
637
+ fit_sampling=self.fit_sampling,
638
+ random_state=self.fit_subsampling_random_state,
639
+ n_waveforms_fit=self.n_waveforms_fit,
640
+ fit_max_reweighting=self.fit_max_reweighting,
641
+ voltages_dataset_name="peeled_voltages_fit",
642
+ waveforms_dataset_name="peeled_waveforms_fit",
643
+ fixed_property_keys=self.fixed_property_keys,
644
+ device=device,
645
+ )
646
+ if not len(waveforms):
647
+ raise ValueError(f"Found no spikes when trying to fit featurizers.")
648
+
649
+ featurization_pipeline = featurization_pipeline.to(device)
650
+ featurization_pipeline.fit(
651
+ recording=self.recording,
652
+ waveforms=waveforms,
653
+ **fixed_properties,
654
+ )
655
+ assert getrefcount(waveforms) == 2
656
+ featurization_pipeline = featurization_pipeline.to("cpu")
657
+ self.featurization_pipeline = featurization_pipeline
658
+ finally:
659
+ self.to("cpu")
660
+ if temp_hdf5_filename.exists():
661
+ temp_hdf5_filename.unlink()
662
+ del waveforms, fixed_properties
663
+ gc.collect()
664
+
665
+ def get_chunk_starts(
666
+ self,
667
+ chunk_starts_samples=None,
668
+ chunk_length_samples=None,
669
+ t_start=None,
670
+ t_end=None,
671
+ subsampled=False,
672
+ n_chunks=None,
673
+ ordered=False,
674
+ skip_last=False,
675
+ ):
676
+ if chunk_starts_samples is not None:
677
+ return chunk_starts_samples
678
+ if chunk_length_samples is None:
679
+ chunk_length_samples = self.chunk_length_samples
680
+
681
+ T_samples = self.recording.get_num_samples()
682
+ if t_end is None:
683
+ t_end = T_samples
684
+ if t_start is None:
685
+ t_start = 0
686
+ chunk_starts_samples = range(t_start, t_end, chunk_length_samples)
687
+ if skip_last:
688
+ chunk_starts_samples = list(chunk_starts_samples)
689
+ if t_end - chunk_starts_samples[-1] < chunk_length_samples:
690
+ chunk_starts_samples = chunk_starts_samples[:-1]
691
+
692
+ if not subsampled:
693
+ return chunk_starts_samples
694
+
695
+ if n_chunks is None:
696
+ chunks_per_second = self.recording.sampling_frequency / chunk_length_samples
697
+ n_chunks = int(np.ceil(self.n_seconds_fit * chunks_per_second))
698
+
699
+ # make a random subset of chunks to use for fitting
700
+ rg = np.random.default_rng(self.fit_subsampling_random_state)
701
+ self.fit_subsampling_random_state = rg
702
+ chunk_starts_samples = rg.choice(
703
+ chunk_starts_samples,
704
+ size=min(len(chunk_starts_samples), n_chunks),
705
+ replace=False,
706
+ )
707
+ if ordered:
708
+ chunk_starts_samples.sort()
709
+ return chunk_starts_samples
710
+
711
+ def run_subsampled_peeling(
712
+ self,
713
+ hdf5_filename,
714
+ chunk_length_samples=None,
715
+ residual_to_h5=False,
716
+ skip_features=False,
717
+ ignore_resuming=False,
718
+ n_chunks=None,
719
+ t_start=None,
720
+ t_end=None,
721
+ computation_cfg=None,
722
+ task_name=None,
723
+ overwrite=True,
724
+ ordered=False,
725
+ skip_last=False,
726
+ show_progress=True,
727
+ ):
728
+ # run peeling on these chunks to the temp folder
729
+ chunk_starts = self.get_chunk_starts(
730
+ subsampled=True,
731
+ chunk_length_samples=chunk_length_samples,
732
+ t_start=t_start,
733
+ t_end=t_end,
734
+ n_chunks=n_chunks,
735
+ ordered=ordered,
736
+ skip_last=skip_last,
737
+ )
738
+ return self.peel(
739
+ hdf5_filename,
740
+ chunk_starts_samples=chunk_starts,
741
+ chunk_length_samples=chunk_length_samples,
742
+ stop_after_n_waveforms=self.max_waveforms_fit,
743
+ ignore_resuming=ignore_resuming,
744
+ residual_to_h5=residual_to_h5,
745
+ skip_features=skip_features,
746
+ computation_cfg=computation_cfg,
747
+ overwrite=overwrite,
748
+ task_name=task_name,
749
+ show_progress=show_progress,
750
+ )
751
+
752
+ def save_models(self, save_folder):
753
+ if self.featurization_pipeline is not None:
754
+ Path(save_folder).mkdir(exist_ok=True)
755
+ torch.save(
756
+ self.featurization_pipeline.state_dict(),
757
+ Path(save_folder) / "featurization_pipeline.pt",
758
+ )
759
+
760
+ def load_models(self, save_folder):
761
+ if not save_folder.exists():
762
+ return
763
+
764
+ feats_pt = Path(save_folder) / "featurization_pipeline.pt"
765
+ if feats_pt.exists():
766
+ assert self.featurization_pipeline is not None
767
+ state_dict = torch.load(feats_pt)
768
+ self.featurization_pipeline.load_state_dict(state_dict)
769
+
770
+ def check_resuming(self, output_hdf5_filename, overwrite=False) -> tuple[int, int]:
771
+ output_hdf5_filename = Path(output_hdf5_filename)
772
+ exists = output_hdf5_filename.exists()
773
+ last_chunk_start = -1
774
+ residual_snips_so_far = 0
775
+ if exists and not overwrite:
776
+ with h5py.File(output_hdf5_filename, "r", locking=False) as h5:
777
+ last_chunk_start: int = h5["last_chunk_start"][()] # type: ignore
778
+ residual_snips_so_far = 0
779
+ if "residual" in h5:
780
+ residual_snips_so_far = len(h5["residual"]) # type: ignore
781
+ return last_chunk_start, residual_snips_so_far
782
+
783
+ @contextmanager
784
+ def initialize_files(
785
+ self,
786
+ output_hdf5_filename,
787
+ residual_filename=None,
788
+ overwrite=False,
789
+ chunk_size=1024,
790
+ # could also do:
791
+ # libver=("earliest", "v110"),
792
+ libver="latest",
793
+ residual_to_h5=False,
794
+ skip_features=False,
795
+ ):
796
+ """Create, overwrite, or re-open output files"""
797
+ if output_hdf5_filename is None:
798
+ assert residual_filename is None
799
+ assert not residual_to_h5
800
+ # this is not usually the case, but it is used by the
801
+ # RunningTemplates peeler.
802
+ yield None, None, None, 0
803
+ return
804
+
805
+ output_hdf5_filename = Path(output_hdf5_filename)
806
+ exists = output_hdf5_filename.exists()
807
+ n_spikes = 0
808
+ if exists and overwrite:
809
+ output_hdf5_filename.unlink()
810
+ output_h5 = h5py.File(output_hdf5_filename, "w", libver=libver)
811
+ output_h5.create_dataset("last_chunk_start", data=-1, dtype=np.int64)
812
+ elif exists:
813
+ # exists and not overwrite
814
+ output_h5 = h5py.File(output_hdf5_filename, "r+", libver=libver)
815
+ n_spikes = len(output_h5["times_samples"]) # type: ignore
816
+ else:
817
+ # didn't exist, so overwrite does not matter
818
+ output_h5 = h5py.File(output_hdf5_filename, "w", libver=libver)
819
+ output_h5.create_dataset("last_chunk_start", data=-1, dtype=np.int64)
820
+ last_chunk_start: int = output_h5["last_chunk_start"][()] # type: ignore
821
+
822
+ # write some fixed arrays that are useful to have around
823
+ for name, value in self.fixed_output_data:
824
+ if name not in output_h5:
825
+ output_h5.create_dataset(name, data=value)
826
+
827
+ # create per-spike datasets
828
+ # use chunks to support growing the dataset as we find spikes
829
+ h5_spike_datasets = None
830
+ if not skip_features:
831
+ h5_spike_datasets = {}
832
+ for ds in self.out_datasets():
833
+ if ds.name in output_h5:
834
+ h5_spike_datasets[ds.name] = output_h5[ds.name]
835
+ else:
836
+ h5_spike_datasets[ds.name] = output_h5.create_dataset(
837
+ ds.name,
838
+ dtype=ds.dtype,
839
+ shape=(n_spikes, *ds.shape_per_spike),
840
+ maxshape=(None, *ds.shape_per_spike),
841
+ chunks=(chunk_size, *ds.shape_per_spike),
842
+ )
843
+
844
+ if residual_to_h5:
845
+ if "residual" not in output_h5:
846
+ n_chans = self.recording.get_num_channels()
847
+ output_h5.create_dataset(
848
+ "residual",
849
+ dtype=self.np_dtype,
850
+ shape=(0, self.spike_length_samples, n_chans),
851
+ maxshape=(None, self.spike_length_samples, n_chans),
852
+ chunks=(chunk_size, self.spike_length_samples, n_chans),
853
+ )
854
+ if "residual_times_seconds" not in output_h5:
855
+ output_h5.create_dataset(
856
+ "residual_times_seconds",
857
+ dtype=float,
858
+ shape=(0,),
859
+ maxshape=(None,),
860
+ chunks=(chunk_size,),
861
+ )
862
+
863
+ # residual file ignore/open/overwrite logic
864
+ save_residual = residual_filename is not None
865
+ residual_file = None
866
+ if save_residual:
867
+ residual_mode = "wb"
868
+ if last_chunk_start >= 0:
869
+ residual_mode = "ab"
870
+ assert Path(residual_filename).exists()
871
+ residual_file = open(residual_filename, mode=residual_mode)
872
+
873
+ try:
874
+ yield (
875
+ output_h5,
876
+ h5_spike_datasets,
877
+ residual_file,
878
+ n_spikes,
879
+ )
880
+ finally:
881
+ self.to("cpu")
882
+ output_h5.close()
883
+ if save_residual:
884
+ assert residual_file is not None
885
+ residual_file.close()
886
+
887
+ # -- thread-local rngs. they're not thread safe, and locals can't be pickled
888
+
889
+ def __getstate__(self):
890
+ state = super().__getstate__()
891
+ del state["_rgs"]
892
+ return state
893
+
894
+ def __setstate__(self, state):
895
+ super().__setstate__(state)
896
+ self._rgs = local()
897
+
898
+ @property
899
+ def rg(self):
900
+ if not hasattr(self._rgs, "rg"):
901
+ with _lock:
902
+ self._rgs.rg = self.fit_subsampling_random_state.spawn(1)[0]
903
+ return self._rgs.rg
904
+
905
+ @staticmethod
906
+ def next_margin(length, factor=10):
907
+ return factor * int(np.ceil(length / factor))
908
+
909
+ def nearest_batch_length(self, target=512):
910
+ factors = divisors(self.chunk_length_samples + 2 * self.chunk_margin_samples)
911
+ factors = np.array(factors)
912
+ return factors[np.abs(factors - target).argmin()]
913
+
914
+
915
+ # -- batch result type stub
916
+
917
+
918
+ try:
919
+ # this isn't supported until 3.15, but it helps the type checker
920
+ # so I'm keeping it in here. everything seems to be fine just setting
921
+ # it to dict in practice in the except clause.
922
+
923
+ class PeelingBatchResult(TypedDict, extra_items=torch.Tensor):
924
+ n_spikes: int
925
+ except TypeError:
926
+ PeelingBatchResult = dict # type: ignore
927
+
928
+
929
+ peeling_empty_result = PeelingBatchResult(n_spikes=0)
930
+
931
+
932
+ # -- helper functions and objects for parallelism
933
+
934
+
935
+ class PeelerProcessContext:
936
+ def __init__(
937
+ self, peeler, compute_residual, skip_features, chunk_length_samples, to_cpu
938
+ ):
939
+ self.peeler = peeler
940
+ self.compute_residual = compute_residual
941
+ self.skip_features = skip_features
942
+ self.chunk_length_samples = chunk_length_samples
943
+ self.to_cpu = to_cpu
944
+
945
+
946
+ # this state will be set on each thread globally
947
+ _peeler_process_context = local()
948
+ _peeler_process_context.ctx = None
949
+
950
+
951
+ def _peeler_process_init(
952
+ peeler,
953
+ device,
954
+ rank_queue,
955
+ compute_residual,
956
+ skip_features,
957
+ chunk_length_samples,
958
+ is_local,
959
+ to_cpu,
960
+ ):
961
+ global _peeler_process_context
962
+
963
+ # figure out what device to work on
964
+ my_rank = rank_queue.get()
965
+ if device is None:
966
+ device = "cuda" if torch.cuda.is_available() else "cpu"
967
+ device = torch.device(device)
968
+ if device.type == "cuda" and device.index is None:
969
+ if torch.cuda.device_count() > 1:
970
+ index = my_rank % torch.cuda.device_count()
971
+ device = torch.device("cuda", index=index)
972
+
973
+ # move peeler to device and put everything in inference mode
974
+ if not is_local:
975
+ peeler.to(device)
976
+ peeler.eval()
977
+
978
+ # update process-local variables
979
+ _peeler_process_context.ctx = PeelerProcessContext(
980
+ peeler, compute_residual, skip_features, chunk_length_samples, to_cpu
981
+ )
982
+
983
+
984
+ def _peeler_process_job(chunk_start_samples__n_resid_snips):
985
+ chunk_start_samples, n_resid_snips = chunk_start_samples__n_resid_snips
986
+ # by returning here, we are implicitly relying on pickle
987
+ # TODO: replace with manual np.saves
988
+ with torch.no_grad():
989
+ chunk_end_samples = None
990
+ chlen = _peeler_process_context.ctx.chunk_length_samples
991
+ if chlen is not None:
992
+ chunk_end_samples = chunk_start_samples + chlen
993
+ return _peeler_process_context.ctx.peeler.process_chunk(
994
+ chunk_start_samples,
995
+ n_resid_snips=n_resid_snips,
996
+ chunk_end_samples=chunk_end_samples,
997
+ return_residual=_peeler_process_context.ctx.compute_residual,
998
+ skip_features=_peeler_process_context.ctx.skip_features,
999
+ to_cpu=_peeler_process_context.ctx.to_cpu,
1000
+ )