DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,840 @@
1
+ from pathlib import Path
2
+ import pickle
3
+ from typing import cast
4
+ import warnings
5
+
6
+ from dredge import motion_util
7
+ import h5py
8
+ import numpy as np
9
+ import pandas as pd
10
+ from scipy.signal import sawtooth
11
+ from spikeinterface.core import read_binary_folder, BaseRecording
12
+ from spikeinterface.core.recording_tools import get_chunk_with_margin
13
+ from spikeinterface.preprocessing.basepreprocessor import (
14
+ BasePreprocessor,
15
+ BasePreprocessorSegment,
16
+ )
17
+ import torch
18
+ from tqdm.auto import tqdm
19
+
20
+ from ..templates import TemplateData
21
+ from ..util.data_util import (
22
+ DARTsortSorting,
23
+ extract_random_snips,
24
+ resolve_path,
25
+ divide_randomly,
26
+ )
27
+ from ..util.multiprocessing_util import get_pool
28
+ from ..util.drift_util import registered_geometry
29
+ from ..util.logging_util import get_logger
30
+ from ..util.spiketorch import ptp
31
+ from ..util.waveform_util import make_channel_index
32
+ from .simlib import simulate_sorting, add_features, default_sim_featurization_cfg
33
+ from .simlib import default_temporal_kernel_npy
34
+ from .noise_recording_tools import get_background_recording
35
+ from .sim_template_tools import get_template_simulator, TemplateSimulator
36
+
37
+
38
+ logger = get_logger(__name__)
39
+
40
+
41
+ def generate_simulation(
42
+ folder: str | Path | None,
43
+ noise_recording_folder: str | Path | None,
44
+ *,
45
+ n_units: int,
46
+ duration_seconds: float,
47
+ # probe parameters
48
+ geom=None,
49
+ probe_kwargs=None,
50
+ # sorting parameters
51
+ firing_kind="uniform",
52
+ min_fr_hz=1.0,
53
+ max_fr_hz=20.0,
54
+ # noise args
55
+ noise_kind="stationary_factorized_rbf",
56
+ noise_spatial_kernel_bandwidth=15.0,
57
+ noise_temporal_kernel: np.ndarray | str | Path = default_temporal_kernel_npy,
58
+ noise_fft_t=121,
59
+ white_noise_scale=1.0,
60
+ # template args
61
+ templates_kind="3exp",
62
+ template_library=None,
63
+ template_simulator_kwargs: dict | None = None,
64
+ template_simulator: TemplateSimulator | None = None,
65
+ # general parameters
66
+ drift_type="triangle",
67
+ drift_speed=0.0,
68
+ drift_period=30.0,
69
+ temporal_jitter=16,
70
+ temporal_jitter_family="uniform",
71
+ amplitude_jitter=0.05,
72
+ amp_jitter_family="uniform",
73
+ common_reference=True,
74
+ sampling_frequency=30000.0,
75
+ refractory_ms=1.0,
76
+ globally_refractory=False,
77
+ extract_radius=100.0,
78
+ recording_dtype="float16",
79
+ features_dtype="float32",
80
+ featurization_cfg=default_sim_featurization_cfg,
81
+ save_injected_waveforms=False,
82
+ save_noise_waveforms=False,
83
+ save_collision_waveforms=False,
84
+ save_collidedness=False,
85
+ # control
86
+ max_drift_per_chunk=0.5,
87
+ max_chunk_len_s=1.0,
88
+ random_seed=0,
89
+ noise_in_memory=False,
90
+ overwrite=False,
91
+ n_jobs=1,
92
+ no_save=False,
93
+ just_noise=False,
94
+ ):
95
+ if folder is not None and not (overwrite or just_noise or no_save):
96
+ try:
97
+ return load_simulation(folder)
98
+ except Exception:
99
+ pass
100
+
101
+ if noise_recording_folder is not None:
102
+ noise_recording_folder = resolve_path(noise_recording_folder)
103
+ else:
104
+ assert noise_in_memory
105
+ duration_samples = int(duration_seconds * sampling_frequency)
106
+ with warnings.catch_warnings(record=True) as ws:
107
+ noise_recording = get_background_recording(
108
+ noise_recording_folder,
109
+ duration_samples=duration_samples,
110
+ geom=geom,
111
+ probe_kwargs=probe_kwargs,
112
+ noise_kind=noise_kind,
113
+ noise_spatial_kernel_bandwidth=noise_spatial_kernel_bandwidth,
114
+ noise_temporal_kernel=noise_temporal_kernel,
115
+ random_seed=random_seed,
116
+ dtype=recording_dtype,
117
+ noise_fft_t=noise_fft_t,
118
+ white_noise_scale=white_noise_scale,
119
+ sampling_frequency=sampling_frequency,
120
+ n_jobs=n_jobs,
121
+ in_memory=noise_in_memory,
122
+ overwrite=overwrite,
123
+ )
124
+ for w in ws:
125
+ msg = str(w.message)
126
+ if msg.startswith("The extractor is not serializable "):
127
+ continue
128
+ if msg.startswith("auto_cast_uint"):
129
+ continue
130
+ raise ValueError(w)
131
+ assert isinstance(noise_recording, BaseRecording)
132
+ assert noise_recording.dtype == np.dtype(recording_dtype)
133
+ assert noise_recording.sampling_frequency == sampling_frequency
134
+ assert noise_recording.get_num_frames() == duration_samples
135
+
136
+ if just_noise:
137
+ return
138
+
139
+ if folder is not None:
140
+ folder = resolve_path(folder)
141
+ else:
142
+ assert no_save
143
+
144
+ if template_simulator is None:
145
+ template_simulator = get_template_simulator(
146
+ n_units=n_units,
147
+ templates_kind=templates_kind,
148
+ template_library=template_library,
149
+ geom=noise_recording.get_channel_locations(),
150
+ sampling_frequency=sampling_frequency,
151
+ common_reference=common_reference,
152
+ random_seed=random_seed,
153
+ temporal_jitter=temporal_jitter,
154
+ **(template_simulator_kwargs or {}),
155
+ )
156
+ sim_recording = InjectSpikesPreprocessor(
157
+ noise_recording,
158
+ firing_kind=firing_kind,
159
+ min_fr_hz=min_fr_hz,
160
+ max_fr_hz=max_fr_hz,
161
+ template_simulator=template_simulator,
162
+ drift_type=drift_type,
163
+ drift_speed=drift_speed,
164
+ drift_period=drift_period,
165
+ amplitude_jitter=amplitude_jitter,
166
+ temporal_jitter_family=temporal_jitter_family,
167
+ temporal_jitter=temporal_jitter,
168
+ random_seed=random_seed,
169
+ refractory_ms=refractory_ms,
170
+ globally_refractory=globally_refractory,
171
+ amp_jitter_family=amp_jitter_family,
172
+ extract_radius=extract_radius,
173
+ features_dtype=features_dtype,
174
+ compute_collision_waveforms=save_collision_waveforms or save_collidedness,
175
+ )
176
+ if no_save:
177
+ return sim_recording, template_simulator
178
+
179
+ if drift_speed is None:
180
+ chunk_len_s = max_chunk_len_s
181
+ else:
182
+ drift_per_chunk = drift_speed * max_chunk_len_s
183
+ chunk_len_s = min(
184
+ max_chunk_len_s, max_drift_per_chunk / max(drift_per_chunk, 1e-10)
185
+ )
186
+ assert chunk_len_s
187
+
188
+ sim_recording.save_simulation(
189
+ folder,
190
+ overwrite=overwrite,
191
+ n_jobs=n_jobs,
192
+ featurization_cfg=featurization_cfg,
193
+ chunk_len_s=chunk_len_s,
194
+ save_injected_waveforms=save_injected_waveforms,
195
+ save_noise_waveforms=save_noise_waveforms,
196
+ save_collision_waveforms=save_collision_waveforms,
197
+ save_collidedness=save_collidedness,
198
+ )
199
+ return load_simulation(folder)
200
+
201
+
202
+ def load_simulation(folder):
203
+ folder = resolve_path(folder, strict=True)
204
+ recording_dir = folder / "recording"
205
+ templates_npz = folder / "templates.npz"
206
+ sorting_h5 = folder / "dartsort_sorting.h5"
207
+ motion_est_pkl = folder / "motion_est.pkl"
208
+ unit_info_csv = folder / "unit_information.csv"
209
+
210
+ recording = read_binary_folder(recording_dir)
211
+ templates = TemplateData.from_npz(templates_npz)
212
+ sorting = DARTsortSorting.from_peeling_hdf5(sorting_h5)
213
+ with open(motion_est_pkl, "rb") as jar:
214
+ motion_est = pickle.load(jar)
215
+ unit_info_df = pd.read_csv(unit_info_csv)
216
+
217
+ return dict(
218
+ recording=recording,
219
+ templates=templates,
220
+ sorting=sorting,
221
+ motion_est=motion_est,
222
+ unit_info_df=unit_info_df,
223
+ )
224
+
225
+
226
+ class InjectSpikesPreprocessor(BasePreprocessor):
227
+ def __init__(self, recording, **simulation_kwargs):
228
+ super().__init__(recording)
229
+
230
+ self._serializability["json"] = False
231
+ self._serializability["pickle"] = False
232
+
233
+ assert len(recording._recording_segments) == 1
234
+ self.add_recording_segment(
235
+ InjectSpikesPreprocessorSegment(
236
+ recording._recording_segments[0],
237
+ n_channels=self.get_num_channels(),
238
+ **simulation_kwargs,
239
+ )
240
+ )
241
+ self.segment = cast(
242
+ InjectSpikesPreprocessorSegment, self._recording_segments[0]
243
+ )
244
+
245
+ def basic_sorting(self) -> DARTsortSorting:
246
+ return self.segment.basic_sorting()
247
+
248
+ def drift(self, t_samples):
249
+ return self.segment.drift(t_samples)
250
+
251
+ def templates(self, t_samples=None, up=False):
252
+ return self.segment.templates(t_samples, up)
253
+
254
+ def motion_estimate(self):
255
+ if not self.segment.drift_speed:
256
+ return None
257
+
258
+ duration_s = np.ceil(self.get_duration())
259
+ t = np.arange(duration_s)
260
+ time_bin_centers = t + 0.5 * np.diff(t).mean()
261
+ tbc_samples = time_bin_centers * self.sampling_frequency
262
+ displacement = self.drift(tbc_samples)
263
+ return motion_util.get_motion_estimate(
264
+ displacement=displacement, time_bin_centers_s=time_bin_centers
265
+ )
266
+
267
+ def registered_geom(self):
268
+ me = self.motion_estimate()
269
+ geom = self.get_channel_locations()
270
+ rgeom = registered_geometry(geom, motion_est=me)
271
+ matches = np.square(geom[None] - rgeom[:, None]).sum(2).argmin(0)
272
+ return rgeom, matches
273
+
274
+ def template_data(self, hdf5_path=None):
275
+ if not self.segment.drift_speed:
276
+ return TemplateData(
277
+ templates=self.templates()[1],
278
+ unit_ids=np.arange(self.segment.n_units),
279
+ spike_counts=np.ones(self.segment.n_units),
280
+ registered_geom=self.get_channel_locations(),
281
+ trough_offset_samples=self.segment.trough_offset_samples,
282
+ )
283
+
284
+ rgeom, matches = self.registered_geom()
285
+ _, templates, _ = self.templates()
286
+ rtemplates = np.zeros(
287
+ (*templates.shape[:-1], len(rgeom)), dtype=templates.dtype
288
+ )
289
+ rtemplates[:, :, matches] = templates
290
+ return TemplateData(
291
+ templates=rtemplates,
292
+ unit_ids=np.arange(self.segment.n_units),
293
+ spike_counts=np.ones(self.segment.n_units),
294
+ registered_geom=rgeom,
295
+ trough_offset_samples=self.segment.trough_offset_samples,
296
+ )
297
+
298
+ def gt_unit_information(self):
299
+ ids = np.arange(self.segment.n_units)
300
+ pos, templates, _ = self.templates()
301
+ x, y, z = pos.T
302
+ counts = np.zeros(self.segment.n_units, dtype=np.int64)
303
+ u, c = np.unique(self.segment.labels, return_counts=True)
304
+ counts[u] = c
305
+ df = dict(
306
+ gt_unit_id=ids,
307
+ x=x,
308
+ y=y,
309
+ z=z,
310
+ ptp_amplitude=np.ptp(templates, axis=1).max(1),
311
+ template_norm=np.linalg.norm(templates, axis=(1, 2)),
312
+ gt_spike_count=counts,
313
+ gt_fr_hz=counts / self.get_total_duration(),
314
+ )
315
+ return pd.DataFrame(df)
316
+
317
+ def save_features_to_hdf5(
318
+ self,
319
+ hdf5_path,
320
+ overwrite=False,
321
+ n_jobs=1,
322
+ show_progress=True,
323
+ n_residual_snips=4096,
324
+ save_injected_waveforms=False,
325
+ save_noise_waveforms=False,
326
+ save_collision_waveforms=False,
327
+ save_collidedness=False,
328
+ chunk_len_s=0.5,
329
+ ):
330
+ if overwrite:
331
+ if hdf5_path.exists():
332
+ hdf5_path.unlink()
333
+ else:
334
+ assert not hdf5_path.exists()
335
+
336
+ n_jobs, Executor, context = get_pool(n_jobs, cls="ThreadPoolExecutor") # type: ignore
337
+ with Executor(max_workers=n_jobs, mp_context=context) as pool:
338
+ nt = self.get_num_frames()
339
+ bs = int(self.sampling_frequency * chunk_len_s)
340
+ chunk_starts = range(0, nt, bs)
341
+ n_residual_snips = min(
342
+ n_residual_snips,
343
+ self.segment.get_num_samples() // self.segment.spike_length_samples,
344
+ )
345
+ residual_snips_per_chunk = divide_randomly(
346
+ n_residual_snips, len(chunk_starts), self.segment.random_seed
347
+ )
348
+ jobs = (
349
+ ((t, min(t + bs, nt)), dict(extract=True, n_residual_snips=nrs))
350
+ for t, nrs in zip(chunk_starts, residual_snips_per_chunk)
351
+ )
352
+ with h5py.File(hdf5_path, "w", locking=False) as h5:
353
+ n = self.segment.n_spikes
354
+
355
+ # fixed arrays
356
+ h5.create_dataset("sampling_frequency", data=self.sampling_frequency)
357
+ h5.create_dataset("geom", data=self.get_channel_locations())
358
+ h5.create_dataset(
359
+ "channel_index", data=self.segment.extract_channel_index
360
+ )
361
+ times_samples = (
362
+ self.segment.times_samples + self.segment.upsampling_offsets
363
+ )
364
+ h5.create_dataset("times_samples", data=times_samples)
365
+ h5.create_dataset(
366
+ "times_seconds",
367
+ data=self.sample_index_to_time(times_samples),
368
+ )
369
+ h5.create_dataset("labels", data=self.segment.labels)
370
+ h5.create_dataset("scalings", data=self.segment.scalings)
371
+ h5.create_dataset("jitter_ix", data=self.segment.jitter_ix)
372
+ pos, temp, off = self.segment.template_simulator.templates(up=True)
373
+ h5.create_dataset("unit_positions", data=pos)
374
+ h5.create_dataset("up_offsets", data=off)
375
+ h5.create_dataset("templates_up", data=temp)
376
+
377
+ # arrays discovered in batches below
378
+ f_dt = self.segment.features_dtype
379
+ inj_wf_shape = self.segment.inj_wf_shape
380
+ dataset_shapes = {
381
+ "localizations": ((3,), f_dt),
382
+ "displacements": ((), f_dt),
383
+ "ptp_amplitudes": ((), f_dt),
384
+ "channels": ((), np.int32),
385
+ "collisioncleaned_waveforms": (inj_wf_shape, f_dt),
386
+ "time_shifts": ((), np.int16),
387
+ }
388
+ if save_injected_waveforms:
389
+ dataset_shapes["injected_waveforms"] = (inj_wf_shape, f_dt)
390
+ if save_noise_waveforms:
391
+ dataset_shapes["noise_waveforms"] = (inj_wf_shape, f_dt)
392
+ if save_collision_waveforms:
393
+ dataset_shapes["collision_waveforms"] = (inj_wf_shape, f_dt)
394
+ if save_collidedness:
395
+ dataset_shapes["collidedness"] = ((), f_dt)
396
+ datasets = {
397
+ k: h5.create_dataset(k, dtype=dt, shape=(n, *sh))
398
+ for k, (sh, dt) in dataset_shapes.items()
399
+ }
400
+
401
+ # residual snippets
402
+ if n_residual_snips:
403
+ residual = h5.create_dataset(
404
+ "residual",
405
+ shape=(n_residual_snips, *self.segment.wf_shape),
406
+ maxshape=(n_residual_snips, *self.segment.wf_shape),
407
+ chunks=(min(16, n_residual_snips), *self.segment.wf_shape),
408
+ dtype=f_dt,
409
+ )
410
+ residual_times = h5.create_dataset(
411
+ "residual_times_seconds",
412
+ shape=(n_residual_snips,),
413
+ maxshape=(n_residual_snips,),
414
+ chunks=(min(16, n_residual_snips),),
415
+ dtype=f_dt,
416
+ )
417
+ else:
418
+ residual = residual_times = None
419
+
420
+ results = pool.map(self.segment._get_traces_and_inject_spikes_job, jobs)
421
+ if show_progress:
422
+ results = tqdm(
423
+ results,
424
+ total=len(chunk_starts),
425
+ desc="Extract GT features",
426
+ smoothing=0.02,
427
+ )
428
+
429
+ i1_prev = 0
430
+ n_injected = 0
431
+ resid_ix = 0
432
+ for res in results:
433
+ assert res is not None
434
+ _, s = res
435
+ del res, _
436
+ i0 = cast(int, s["i0"])
437
+ i1 = cast(int, s["i1"])
438
+ assert i0 == i1_prev
439
+ i1_prev = i1
440
+ n_injected += i1 - i0
441
+
442
+ for k, ds in datasets.items():
443
+ ds[i0:i1] = s[k]
444
+
445
+ nrs = s["n_residual_snips"]
446
+ if not nrs:
447
+ continue
448
+ assert nrs == cast(np.ndarray, s["residual"]).shape[0]
449
+ assert nrs == cast(np.ndarray, s["residual_times"]).shape[0]
450
+ assert residual is not None
451
+ assert residual_times is not None
452
+ residual[resid_ix : resid_ix + nrs] = s["residual"]
453
+ residual_times[resid_ix : resid_ix + nrs] = s["residual_times"]
454
+ resid_ix += nrs
455
+ if residual is not None and resid_ix != n_residual_snips:
456
+ assert residual_times is not None
457
+ residual.resize((resid_ix, *residual.shape[1:]))
458
+ residual_times.resize((resid_ix, *residual_times.shape[1:]))
459
+ if residual is not None:
460
+ assert residual_times is not None
461
+ assert residual.shape[0] == residual_times.shape[0] == resid_ix
462
+ assert i1_prev == n
463
+ assert n_injected == n
464
+
465
+ def save_simulation(
466
+ self,
467
+ folder,
468
+ overwrite=False,
469
+ n_jobs=1,
470
+ featurization_cfg=default_sim_featurization_cfg,
471
+ save_injected_waveforms=False,
472
+ save_noise_waveforms=False,
473
+ save_collision_waveforms=False,
474
+ save_collidedness=False,
475
+ chunk_len_s=0.5,
476
+ ):
477
+ folder = resolve_path(folder)
478
+ folder.mkdir(exist_ok=True)
479
+ recording_dir = folder / "recording"
480
+ templates_npz = folder / "templates.npz"
481
+ sorting_h5 = folder / "dartsort_sorting.h5"
482
+ motion_est_pkl = folder / "motion_est.pkl"
483
+ unit_info_csv = folder / "unit_information.csv"
484
+
485
+ with warnings.catch_warnings(record=True) as ws:
486
+ recording = self.save_to_folder(
487
+ folder=recording_dir,
488
+ overwrite=overwrite, # type: ignore
489
+ n_jobs=n_jobs,
490
+ pool_engine="thread",
491
+ chunk_duration=chunk_len_s,
492
+ )
493
+ for w in ws:
494
+ msg = str(w.message)
495
+ if msg.startswith("The extractor is not serializable "):
496
+ continue
497
+ if msg.startswith("auto_cast_uint"):
498
+ continue
499
+ raise w.category(w.message)
500
+ n_residual_snips = (
501
+ 0 if featurization_cfg is None else featurization_cfg.n_residual_snips
502
+ )
503
+ self.save_features_to_hdf5(
504
+ sorting_h5,
505
+ n_jobs=n_jobs,
506
+ overwrite=overwrite,
507
+ n_residual_snips=n_residual_snips,
508
+ save_injected_waveforms=save_injected_waveforms,
509
+ save_noise_waveforms=save_noise_waveforms,
510
+ save_collision_waveforms=save_collision_waveforms,
511
+ save_collidedness=save_collidedness,
512
+ chunk_len_s=chunk_len_s,
513
+ )
514
+ if featurization_cfg is not None and not featurization_cfg.skip:
515
+ # this is only for the TPCA feature.
516
+ torch.manual_seed(self.segment.random_seed)
517
+ add_features(sorting_h5, recording, featurization_cfg)
518
+
519
+ self.gt_unit_information().to_csv(unit_info_csv)
520
+ with open(motion_est_pkl, "wb") as jar:
521
+ pickle.dump(self.motion_estimate(), jar)
522
+ self.template_data(sorting_h5).to_npz(templates_npz)
523
+
524
+
525
+ class InjectSpikesPreprocessorSegment(BasePreprocessorSegment):
526
+ def __init__(
527
+ self,
528
+ parent_recording_segment,
529
+ n_channels,
530
+ *,
531
+ firing_kind,
532
+ min_fr_hz,
533
+ max_fr_hz,
534
+ template_simulator,
535
+ drift_type,
536
+ drift_speed,
537
+ drift_period,
538
+ amplitude_jitter,
539
+ amp_jitter_family,
540
+ temporal_jitter,
541
+ temporal_jitter_family,
542
+ random_seed,
543
+ refractory_ms,
544
+ globally_refractory,
545
+ extract_radius,
546
+ features_dtype="float32",
547
+ compute_collision_waveforms=False,
548
+ ):
549
+ super().__init__(parent_recording_segment)
550
+ assert self.sampling_frequency is not None
551
+ assert drift_type in ("line", "triangle")
552
+
553
+ self.drift_type = drift_type
554
+ self.drift_speed = drift_speed
555
+ self.drift_period = drift_period
556
+ self.temporal_jitter = temporal_jitter
557
+ self.temporal_jitter_family = temporal_jitter_family
558
+ self.features_dtype = features_dtype
559
+ self.template_simulator = template_simulator
560
+ self.refractory_samples = int(
561
+ refractory_ms * (self.sampling_frequency / 1000.0)
562
+ )
563
+
564
+ # shapes
565
+ self.n_units = self.template_simulator.n_units
566
+ self.n_channels = n_channels
567
+ self.chans_arange = np.arange(n_channels)
568
+ self.spike_length_samples = self.template_simulator.spike_length_samples()
569
+ self.trough_offset_samples = self.template_simulator.trough_offset_samples()
570
+ self.snippet_time_ix = np.arange(
571
+ -self.trough_offset_samples,
572
+ self.spike_length_samples - self.trough_offset_samples,
573
+ )
574
+ self.spike_safe_pad = max(
575
+ self.trough_offset_samples,
576
+ self.spike_length_samples - self.trough_offset_samples,
577
+ )
578
+ self.margin = 2 * self.spike_safe_pad
579
+ self.extract_channel_index = make_channel_index(
580
+ self.template_simulator.geom, extract_radius
581
+ )
582
+ self.wf_shape = (self.spike_length_samples, self.n_channels)
583
+ self.inj_wf_shape = (
584
+ self.spike_length_samples,
585
+ self.extract_channel_index.shape[1],
586
+ )
587
+ self.template_shape = (self.n_units, *self.wf_shape)
588
+ self.template_shape_up = (self.n_units, temporal_jitter, *self.wf_shape)
589
+ self.compute_collision_waveforms = compute_collision_waveforms
590
+
591
+ # bake all random stuff
592
+ rg = np.random.default_rng(random_seed)
593
+ if firing_kind == "uniform":
594
+ firing_rates = rg.uniform(min_fr_hz, max_fr_hz, size=self.n_units)
595
+ else:
596
+ assert False
597
+ self.random_seed = random_seed
598
+ sorting = simulate_sorting(
599
+ self.n_units,
600
+ self.get_num_samples(),
601
+ firing_rates=firing_rates,
602
+ rg=rg,
603
+ nbefore=self.trough_offset_samples,
604
+ spike_length_samples=self.spike_length_samples,
605
+ sampling_frequency=self.sampling_frequency,
606
+ refractory_samples=self.refractory_samples,
607
+ globally_refractory=globally_refractory,
608
+ )
609
+ sv = cast(np.recarray, sorting.to_spike_vector())
610
+ self.times_samples = sv["sample_index"]
611
+ self.labels = sv["unit_index"]
612
+ self.n_spikes = sorting.count_total_num_spikes()
613
+ assert self.times_samples.shape == self.labels.shape == (self.n_spikes,)
614
+
615
+ if amplitude_jitter and amp_jitter_family == "gamma":
616
+ alpha = 1.0 / amplitude_jitter**2
617
+ theta = amplitude_jitter**2
618
+ self.scalings = rg.gamma(shape=alpha, scale=theta, size=self.n_spikes)
619
+ elif amplitude_jitter and amp_jitter_family == "uniform":
620
+ self.scalings = rg.uniform(
621
+ 1.0 - amplitude_jitter, 1.0 + amplitude_jitter, size=self.n_spikes
622
+ )
623
+ else:
624
+ assert not amplitude_jitter
625
+ self.scalings = np.ones(1, dtype=features_dtype)
626
+ self.scalings = np.broadcast_to(self.scalings, (self.n_spikes,))
627
+
628
+ if self.temporal_jitter > 1 and temporal_jitter_family == "uniform":
629
+ self.jitter_ix = rg.integers(self.temporal_jitter, size=self.n_spikes)
630
+ elif self.temporal_jitter > 1 and temporal_jitter_family == "by_unit":
631
+ self.jitter_ix = self.labels % self.temporal_jitter
632
+ else:
633
+ self.jitter_ix = np.zeros(1, dtype=np.int64)
634
+ self.jitter_ix = np.broadcast_to(self.jitter_ix, (self.n_spikes,))
635
+ if temporal_jitter == 1:
636
+ self.upsampling_offsets = np.zeros_like(self.times_samples)
637
+ else:
638
+ self.upsampling_offsets = template_simulator.offsets_up[
639
+ self.labels, self.jitter_ix
640
+ ]
641
+
642
+ def basic_sorting(self) -> DARTsortSorting:
643
+ assert isinstance(self.sampling_frequency, (int, float))
644
+ return DARTsortSorting(
645
+ times_samples=self.times_samples + self.upsampling_offsets,
646
+ labels=self.labels,
647
+ channels=np.zeros_like(self.times_samples),
648
+ sampling_frequency=self.sampling_frequency,
649
+ ephemeral_features=dict(
650
+ time_shifts=self.upsampling_offsets,
651
+ jitter_ix=self.jitter_ix,
652
+ scalings=self.scalings,
653
+ ),
654
+ )
655
+
656
+ def drift(self, t_samples):
657
+ if not self.drift_speed:
658
+ return np.zeros_like(t_samples)
659
+
660
+ if self.drift_type == "line":
661
+ t_center = self.get_num_samples() / 2
662
+ dt = (t_samples - t_center) / self.sampling_frequency
663
+ return dt * self.drift_speed
664
+
665
+ if self.drift_type == "triangle":
666
+ t_seconds = self.sample_index_to_time(t_samples)
667
+ phase = t_seconds * (2 * np.pi / self.drift_period)
668
+ wave = sawtooth(phase, width=0.5) # type: ignore
669
+ # -1 to 1 and back to -1, so divide by 4 to have 2*ptp=drift_speed*drift_period.
670
+ return wave * (self.drift_speed * self.drift_period / 4.0)
671
+
672
+ assert False
673
+
674
+ def templates(self, t_samples=None, up=False, padded=False, pad_value=np.nan):
675
+ drift = 0 if t_samples is None else self.drift(t_samples)
676
+ pos, templates, offsets = self.template_simulator.templates(
677
+ drift=drift, up=up, padded=padded, pad_value=pad_value
678
+ )
679
+ templates = templates.astype(self.features_dtype)
680
+ tunpad = templates[..., :-1] if padded else templates
681
+ if up:
682
+ assert tunpad.shape == self.template_shape_up
683
+ else:
684
+ assert tunpad.shape == self.template_shape
685
+
686
+ return pos, templates, offsets
687
+
688
+ def get_spikes(
689
+ self,
690
+ noise_with_margin,
691
+ start_frame,
692
+ end_frame,
693
+ in_chunk_only=True,
694
+ extract=False,
695
+ n_residual_snips=0,
696
+ ):
697
+ if in_chunk_only:
698
+ search_start = start_frame
699
+ search_end = end_frame
700
+ else:
701
+ search_start = start_frame - self.spike_safe_pad
702
+ search_end = end_frame + self.spike_safe_pad
703
+ i0 = np.searchsorted(self.times_samples, search_start)
704
+ i1 = np.searchsorted(self.times_samples, search_end)
705
+ t = self.times_samples[i0:i1]
706
+ l = self.labels[i0:i1]
707
+ s = self.scalings[i0:i1]
708
+ u = self.jitter_ix[i0:i1]
709
+
710
+ # temporal indices of snippets relative to chunk
711
+ t_rel = t - (start_frame - self.margin)
712
+ tix = t_rel[:, None] + self.snippet_time_ix
713
+ tc = (start_frame + end_frame) / 2
714
+ pos, temps, offsets = self.templates(t_samples=tc, up=True, padded=extract)
715
+ temps = temps[l, u]
716
+ temps *= s[:, None, None]
717
+ temps_unpad = temps[..., :-1] if extract else temps
718
+ offsets = offsets[l, u]
719
+
720
+ spikes = dict(
721
+ i0=i0,
722
+ i1=i1,
723
+ times_samples=t + offsets,
724
+ time_shifts=offsets.astype(np.int16),
725
+ labels=l,
726
+ scalings=s,
727
+ jitter_ix=u,
728
+ tix=tix,
729
+ waveforms=temps_unpad,
730
+ n_residual_snips=n_residual_snips,
731
+ )
732
+ if not extract:
733
+ return spikes
734
+
735
+ spikes["localizations"] = pos[l]
736
+ spikes["displacements"] = np.full(
737
+ i1 - i0, self.drift(tc), dtype=self.features_dtype
738
+ )
739
+ ptp_vectors = ptp(temps_unpad, dim=1)
740
+ c = ptp_vectors.argmax(axis=1)
741
+ spikes["channels"] = c
742
+ spikes["ptp_amplitudes"] = ptp_vectors.max(axis=1)
743
+ if n_residual_snips:
744
+ rsnips, rtimes = extract_random_snips(
745
+ rg=self.random_seed,
746
+ chunk=noise_with_margin[
747
+ self.margin : len(noise_with_margin) - self.margin
748
+ ],
749
+ n=n_residual_snips,
750
+ sniplen=self.spike_length_samples,
751
+ )
752
+ if torch.is_tensor(rsnips):
753
+ rsnips = rsnips.numpy(force=True)
754
+ spikes["residual"] = rsnips
755
+ rtimes += start_frame
756
+ rtimes = self.sample_index_to_time(rtimes)
757
+ spikes["residual_times"] = rtimes
758
+ spikes["n_residual_snips"] = rtimes.shape[0]
759
+
760
+ # extract the background noise which waveforms will be added into
761
+ noise_padded = np.pad(
762
+ noise_with_margin, [(0, 0), (0, 1)], constant_values=np.nan
763
+ )
764
+ echans = self.extract_channel_index[c]
765
+ if torch.is_tensor(echans):
766
+ echans = echans.numpy(force=True)
767
+ noise_waveforms = noise_padded[tix[:, :, None], echans[:, None, :]]
768
+ # the actual injected waveforms...
769
+ injected_waveforms = np.take_along_axis(temps, echans[:, None, :], axis=2)
770
+ collisioncleaned_waveforms = noise_waveforms + injected_waveforms
771
+
772
+ spikes["noise_waveforms"] = noise_waveforms
773
+ spikes["injected_waveforms"] = injected_waveforms
774
+ spikes["collisioncleaned_waveforms"] = collisioncleaned_waveforms
775
+ spikes["echans"] = echans
776
+
777
+ return spikes
778
+
779
+ def get_traces_and_inject_spikes(
780
+ self,
781
+ start_frame,
782
+ end_frame,
783
+ channel_indices=None,
784
+ extract=False,
785
+ inject=False,
786
+ n_residual_snips=0,
787
+ ):
788
+ traces, lm, rm = get_chunk_with_margin(
789
+ self.parent_recording_segment,
790
+ start_frame,
791
+ end_frame,
792
+ margin=self.margin,
793
+ add_zeros=True,
794
+ channel_indices=None,
795
+ )
796
+ assert lm == rm == self.margin
797
+ assert traces.shape[1] == self.n_channels
798
+ assert extract != inject
799
+ spikes = self.get_spikes(
800
+ traces,
801
+ start_frame,
802
+ end_frame,
803
+ in_chunk_only=not inject,
804
+ extract=extract,
805
+ n_residual_snips=n_residual_snips,
806
+ )
807
+
808
+ if not inject and not self.compute_collision_waveforms:
809
+ return traces, spikes
810
+
811
+ waveforms = cast(np.ndarray, spikes["waveforms"])
812
+ tix = cast(np.ndarray, spikes["tix"])
813
+ waveforms = waveforms.astype(traces.dtype, copy=False)
814
+ traces = traces.copy()
815
+ np.add.at(traces, (tix[:, :, None], self.chans_arange[None, None]), waveforms)
816
+
817
+ if self.compute_collision_waveforms and not inject:
818
+ echans = cast(np.ndarray, spikes["echans"])
819
+ traces_pad = np.pad(traces, [(0, 0), (0, 1)], constant_values=np.nan)
820
+ cwfs = traces_pad[tix[:, :, None], echans[:, None, :]]
821
+ cwfs -= spikes["collisioncleaned_waveforms"]
822
+ spikes["collision_waveforms"] = cwfs
823
+ cwfs = np.nan_to_num(cwfs).reshape(len(cwfs), -1)
824
+ spikes["collidedness"] = np.linalg.norm(cwfs, axis=1)
825
+
826
+ traces = traces[self.margin : len(traces) - self.margin]
827
+ if channel_indices is not None:
828
+ traces = traces[:, channel_indices]
829
+
830
+ return traces, spikes
831
+
832
+ def _get_traces_and_inject_spikes_job(self, args_kwargs):
833
+ args, kwargs = args_kwargs
834
+ return self.get_traces_and_inject_spikes(*args, **kwargs)
835
+
836
+ def get_traces(self, start_frame, end_frame, channel_indices):
837
+ traces, _ = self.get_traces_and_inject_spikes(
838
+ start_frame, end_frame, channel_indices, inject=True
839
+ )
840
+ return traces