DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,594 @@
1
+ """Deeper object-oriented interaction with sorter data
2
+
3
+ This is meant to make implementing plotting code easier: this
4
+ code becomes the model in a MVC framework, and vis/unit.py can
5
+ implement a view and controller.
6
+
7
+ This should also make it easier to compute drift-aware metrics
8
+ (e.g., d' using registered templates and shifted waveforms).
9
+ """
10
+
11
+ from dataclasses import dataclass
12
+
13
+ import numpy as np
14
+ import spikeinterface.core as sc
15
+ import torch
16
+ from dredge.motion_util import MotionEstimate
17
+ from sklearn.decomposition import PCA
18
+
19
+ from ..clustering import merge
20
+ from ..util.internal_config import (
21
+ TemplateConfig,
22
+ TemplateMergeConfig,
23
+ ComputationConfig,
24
+ ClusteringFeaturesConfig,
25
+ default_clustering_features_cfg,
26
+ )
27
+ from ..templates import TemplateData
28
+ from ..util.data_util import (
29
+ DARTsortSorting,
30
+ get_tpca,
31
+ try_get_model_dir,
32
+ sorting_isis,
33
+ )
34
+ from ..util.drift_util import (
35
+ get_spike_pitch_shifts,
36
+ get_waveforms_on_static_channels,
37
+ get_stable_channels,
38
+ )
39
+ from ..util.interpolation_util import StableFeaturesInterpolator, pad_geom
40
+ from ..util.py_util import databag
41
+ from ..util.spikeio import read_waveforms_channel_index
42
+ from ..util.waveform_util import make_channel_index
43
+ from ..util import job_util, logging_util
44
+
45
+
46
+ logger = logging_util.get_logger(__name__)
47
+
48
+
49
+ @dataclass
50
+ class DARTsortAnalysis:
51
+ """Stores all relevant properties for a drift-aware waveform analysis"""
52
+
53
+ sorting: DARTsortSorting
54
+ recording: sc.BaseRecording
55
+ template_data: TemplateData | None
56
+ coarse_template_data: TemplateData | None
57
+ motion_est: MotionEstimate | None
58
+ merge_distances: np.ndarray | None
59
+ geom: np.ndarray
60
+ registered_geom: np.ndarray
61
+ extract_channel_index: np.ndarray | None
62
+ vis_channel_index: np.ndarray
63
+ xyza: np.ndarray | None
64
+ x: np.ndarray | None
65
+ z: np.ndarray | None
66
+ registered_z: np.ndarray | None
67
+ shifting: bool
68
+ times_seconds: np.ndarray | None
69
+ amplitudes: np.ndarray | None
70
+ unit_ids: np.ndarray
71
+ spike_counts: np.ndarray
72
+ amplitude_vectors: np.ndarray | None
73
+ erp: StableFeaturesInterpolator | None
74
+ sklearn_tpca: PCA | None
75
+ tpca_temporal_slice: slice
76
+ name: str | None = None
77
+ vis_radius: float = 50.0
78
+ trough_offset_samples: int = 42
79
+ spike_length_samples: int = 121
80
+ tpca_features_dset: str = "collisioncleaned_tpca_features"
81
+
82
+ @classmethod
83
+ def from_sorting(
84
+ cls,
85
+ recording: sc.BaseRecording,
86
+ sorting: DARTsortSorting,
87
+ motion_est=None,
88
+ name: str | None = None,
89
+ template_data: TemplateData | None = None,
90
+ template_cfg: TemplateConfig | None = TemplateConfig(denoising_method="none"),
91
+ template_merge_cfg: TemplateMergeConfig = TemplateMergeConfig(
92
+ min_spatial_cosine=0.8
93
+ ),
94
+ clustering_features_cfg: ClusteringFeaturesConfig = default_clustering_features_cfg,
95
+ computation_cfg: ComputationConfig | None = None,
96
+ vis_radius: float = 50.0,
97
+ vis_neighborhood_p: float = np.inf,
98
+ ):
99
+ """Try to re-load as much info as possible from the sorting itself
100
+
101
+ Templates are re-computed if labels are not the same as in h5
102
+ or if the template npz does not exist.
103
+ """
104
+ computation_cfg = job_util.ensure_computation_config(computation_cfg)
105
+ has_hdf5 = sorting.parent_h5_path is not None
106
+
107
+ if has_hdf5 and vis_radius and (tpca := get_tpca(sorting)) is not None:
108
+ sklearn_tpca = tpca.to_sklearn() # type: ignore
109
+ tpca_temporal_slice = sklearn_tpca.temporal_slice
110
+ else:
111
+ sklearn_tpca = None
112
+ tpca_temporal_slice = slice(None)
113
+
114
+ model_dir = try_get_model_dir(sorting)
115
+ if template_data is not None:
116
+ pass
117
+ elif has_hdf5 and model_dir is not None:
118
+ template_npz = model_dir / "template_data.npz"
119
+ can_reload = sorting.has_persistent_labels()
120
+ if can_reload and template_npz.exists():
121
+ logger.info(f"Reloading templates from {template_npz}...")
122
+ template_data = TemplateData.from_npz(template_npz)
123
+
124
+ if template_data is None and template_cfg is not None:
125
+ template_data = TemplateData.from_config(
126
+ recording,
127
+ sorting,
128
+ template_cfg,
129
+ overwrite=False,
130
+ motion_est=motion_est,
131
+ computation_cfg=computation_cfg,
132
+ )
133
+
134
+ if template_data is not None:
135
+ coarse_template_data = template_data.coarsen()
136
+ merge_distances = merge.get_merge_distances(
137
+ template_data=coarse_template_data,
138
+ template_merge_cfg=template_merge_cfg,
139
+ computation_cfg=computation_cfg,
140
+ sampling_frequency=recording.sampling_frequency,
141
+ )[1]
142
+ trough_offset_samples = template_data.trough_offset_samples
143
+ spike_length_samples = template_data.spike_length_samples
144
+ else:
145
+ trough_offset_samples = spike_length_samples = 0
146
+ coarse_template_data = merge_distances = None
147
+
148
+ channel_index = getattr(sorting, "channel_index", None)
149
+ amplitudes = getattr(
150
+ sorting, clustering_features_cfg.amplitudes_dataset_name, None
151
+ )
152
+ amplitude_vecs = getattr(
153
+ sorting, clustering_features_cfg.amplitude_vectors_dataset_name, None
154
+ )
155
+ xyza = getattr(
156
+ sorting, clustering_features_cfg.localizations_dataset_name, None
157
+ )
158
+ tpca_features_dset = clustering_features_cfg.pca_dataset_name
159
+ times_seconds = getattr(sorting, "times_seconds", None)
160
+ assert times_seconds is not None
161
+ if motion_est is None and xyza is not None:
162
+ reg_z = xyza[:, 2]
163
+ elif xyza is not None:
164
+ assert motion_est is not None
165
+ reg_z = motion_est.correct_s(times_seconds, xyza[:, 2])
166
+ else:
167
+ reg_z = None
168
+ if xyza is not None:
169
+ x = xyza[:, 0]
170
+ z = xyza[:, 2]
171
+ else:
172
+ x = z = None
173
+
174
+ geom = recording.get_channel_locations()
175
+ if motion_est is None:
176
+ rgeom = geom
177
+ if template_data is not None:
178
+ trg = template_data.registered_geom
179
+ assert (trg is None) or np.array_equal(rgeom, trg)
180
+ else:
181
+ assert template_data is not None
182
+ rgeom = template_data.registered_geom
183
+ assert rgeom is not None
184
+
185
+ device = computation_cfg.actual_device()
186
+ if vis_radius and channel_index is not None:
187
+ # interping to geom with shifts on fly rather than rgeom.
188
+ erp = StableFeaturesInterpolator(
189
+ source_geom=pad_geom(geom, device=device),
190
+ target_geom=pad_geom(geom, device=device),
191
+ channel_index=torch.asarray(channel_index, device=device),
192
+ params=clustering_features_cfg.interp_params,
193
+ )
194
+ else:
195
+ erp = None
196
+
197
+ unit_ids, spike_counts = np.unique(sorting.labels, return_counts=True) # type: ignore
198
+ spike_counts = spike_counts[unit_ids >= 0]
199
+ unit_ids = unit_ids[unit_ids >= 0]
200
+
201
+ return cls(
202
+ sorting=sorting,
203
+ recording=recording,
204
+ template_data=template_data,
205
+ coarse_template_data=coarse_template_data,
206
+ motion_est=motion_est,
207
+ merge_distances=merge_distances,
208
+ geom=geom,
209
+ registered_geom=rgeom,
210
+ extract_channel_index=channel_index,
211
+ vis_channel_index=make_channel_index(
212
+ geom=rgeom, radius=vis_radius, p=vis_neighborhood_p, to_torch=False
213
+ ),
214
+ xyza=xyza,
215
+ x=x,
216
+ z=z,
217
+ registered_z=reg_z,
218
+ shifting=motion_est is not None,
219
+ times_seconds=times_seconds,
220
+ amplitudes=amplitudes,
221
+ amplitude_vectors=amplitude_vecs,
222
+ erp=erp,
223
+ sklearn_tpca=sklearn_tpca,
224
+ tpca_temporal_slice=tpca_temporal_slice,
225
+ name=name,
226
+ vis_radius=vis_radius,
227
+ trough_offset_samples=trough_offset_samples,
228
+ spike_length_samples=spike_length_samples,
229
+ tpca_features_dset=tpca_features_dset,
230
+ unit_ids=unit_ids,
231
+ spike_counts=spike_counts,
232
+ )
233
+
234
+ def in_unit(self, unit_id):
235
+ assert self.sorting.labels is not None
236
+ return np.flatnonzero(np.isin(self.sorting.labels, unit_id))
237
+
238
+ def in_template(self, template_index):
239
+ template_indices = getattr(self.sorting, "template_inds", None)
240
+ assert template_indices is not None
241
+ return np.flatnonzero(np.isin(template_indices, template_index))
242
+
243
+ def unit_template_indices(self, unit_id):
244
+ assert self.template_data is not None
245
+ return np.flatnonzero(self.template_data.unit_ids == unit_id)
246
+
247
+ def unit_amplitudes(self, unit_id=None):
248
+ assert self.template_data is not None
249
+ if unit_id is not None:
250
+ return np.ptp(self.template_data.unit_templates(unit_id))
251
+ amplitudes = np.zeros(self.sorting.unit_ids.shape)
252
+ for j, unit_id in enumerate(self.sorting.unit_ids):
253
+ temps = self.template_data.unit_templates(unit_id)
254
+ amplitudes[j] = np.ptp(np.nan_to_num(temps))
255
+ return amplitudes
256
+
257
+ def firing_rates(self):
258
+ assert self.sorting.labels is not None
259
+ assert np.array_equal(self.unit_ids, self.sorting.unit_ids)
260
+ frs = self.spike_counts / self.recording.get_duration()
261
+ return frs
262
+
263
+ def named_feature(self, fname: str, which: slice | np.ndarray):
264
+ if hasattr(self, fname):
265
+ return getattr(self, fname)[which]
266
+ else:
267
+ return self.sorting.slice_feature_by_name(fname, mask=which)
268
+
269
+ # cluster-dependent feature loading methods
270
+
271
+ def unit_raw_waveforms(
272
+ self,
273
+ unit_id: int | None = None,
274
+ which: np.ndarray | None = None,
275
+ template_index: int | None = None,
276
+ max_count=250,
277
+ main_channel=None,
278
+ random_seed=0,
279
+ dtype=np.float32,
280
+ ) -> "WaveformsBag | None":
281
+ """Load raw waveforms for visualization"""
282
+ if which is not None:
283
+ pass
284
+ elif template_index is not None:
285
+ assert template_index in self.unit_template_indices(unit_id)
286
+ which = self.in_template(template_index)
287
+ elif unit_id is not None:
288
+ which = self.in_unit(unit_id)
289
+ else:
290
+ assert False
291
+
292
+ if max_count is not None and which.size > max_count:
293
+ rg = np.random.default_rng(random_seed)
294
+ which = rg.choice(which, size=max_count, replace=False)
295
+ which.sort()
296
+
297
+ times = self.sorting.times_samples[which]
298
+ mnt = self.trough_offset_samples
299
+ mxt = self.recording.get_num_samples() - self.spike_length_samples
300
+ which = which[times == times.clip(mnt, mxt)]
301
+
302
+ if not which.size:
303
+ return None
304
+
305
+ # read waveforms from disk
306
+ read_chans = self.sorting.channels[which]
307
+ if self.extract_channel_index is None:
308
+ read_channel_index = self.vis_channel_index
309
+ else:
310
+ read_channel_index = self.extract_channel_index
311
+ waveforms = read_waveforms_channel_index(
312
+ recording=self.recording,
313
+ times_samples=self.sorting.times_samples[which],
314
+ channel_index=read_channel_index,
315
+ main_channels=read_chans,
316
+ trough_offset_samples=self.trough_offset_samples,
317
+ spike_length_samples=self.spike_length_samples,
318
+ fill_value=np.nan,
319
+ )
320
+ waveforms = waveforms.astype(dtype)
321
+
322
+ waveforms, main_channel = self.unit_select_channels(
323
+ unit_id=unit_id,
324
+ which=which,
325
+ waveforms=waveforms,
326
+ read_chans=read_chans,
327
+ main_channel=main_channel,
328
+ )
329
+ return WaveformsBag(
330
+ which=which,
331
+ waveforms=waveforms,
332
+ main_channel=main_channel,
333
+ geom=self.registered_geom,
334
+ channel_index=self.vis_channel_index,
335
+ temporal_slice=None,
336
+ )
337
+
338
+ def tpca_features(self, which: np.ndarray):
339
+ assert self.erp is not None
340
+ features = self.sorting.slice_feature_by_name(
341
+ self.tpca_features_dset, mask=which
342
+ )
343
+ device = self.erp.b.source_geom.device
344
+ channels = torch.asarray(self.sorting.channels[which], device=device)
345
+ if self.motion_est is None:
346
+ shifts = torch.zeros(channels.shape, device=device)
347
+ else:
348
+ assert self.z is not None
349
+ assert self.registered_z is not None
350
+ # target will be geom - shift
351
+ # want to move from original position to reg pos = z - disp
352
+ # so, should take shift=disp=z-reg_z
353
+ shifts = self.z[which] - self.registered_z[which]
354
+ shifts = torch.asarray(shifts, device=device).float()
355
+ features = self.erp.interp(
356
+ features=torch.asarray(features, device=device),
357
+ source_main_channels=channels,
358
+ target_channels=self.erp.b.channel_index[channels],
359
+ source_shifts=shifts,
360
+ )
361
+ return features.numpy(force=True)
362
+
363
+ def unit_tpca_waveforms(
364
+ self,
365
+ unit_id,
366
+ which: np.ndarray | None = None,
367
+ template_index=None,
368
+ max_count=250,
369
+ random_seed=0,
370
+ ) -> "WaveformsBag | None":
371
+ assert self.sklearn_tpca is not None
372
+ if which is not None:
373
+ pass
374
+ elif template_index is not None:
375
+ assert template_index in self.unit_template_indices(unit_id)
376
+ which = self.in_template(template_index)
377
+ elif unit_id is not None:
378
+ which = self.in_unit(unit_id)
379
+ else:
380
+ assert False
381
+
382
+ if max_count is not None and which.size > max_count:
383
+ rg = np.random.default_rng(random_seed)
384
+ which = rg.choice(which, size=max_count, replace=False)
385
+ which.sort()
386
+
387
+ if not which.size:
388
+ return None
389
+
390
+ tpca_embeds = self.tpca_features(which=which)
391
+ n, rank, c = tpca_embeds.shape
392
+ tpca_embeds = tpca_embeds.transpose(0, 2, 1).reshape(n * c, rank)
393
+ waveforms = np.full(
394
+ (n * c, self.sklearn_tpca.components_.shape[1]),
395
+ np.nan,
396
+ dtype=tpca_embeds.dtype,
397
+ )
398
+ valid = np.flatnonzero(np.isfinite(tpca_embeds[:, 0]))
399
+ waveforms[valid] = self.sklearn_tpca.inverse_transform(tpca_embeds[valid])
400
+ t = waveforms.shape[1]
401
+ waveforms = waveforms.reshape(n, c, t).transpose(0, 2, 1)
402
+
403
+ waveforms, main_channel = self.unit_select_channels(
404
+ unit_id=unit_id, which=which, waveforms=waveforms
405
+ )
406
+ return WaveformsBag(
407
+ which=which,
408
+ waveforms=waveforms,
409
+ main_channel=main_channel,
410
+ geom=self.registered_geom,
411
+ channel_index=self.vis_channel_index,
412
+ temporal_slice=self.tpca_temporal_slice,
413
+ )
414
+
415
+ def unit_pca_features(
416
+ self,
417
+ unit_id,
418
+ rank=2,
419
+ random_seed=0,
420
+ max_count=500,
421
+ max_wfs_fit=10_000,
422
+ random_state=0,
423
+ ):
424
+ tpca_waves = self.unit_tpca_waveforms(
425
+ unit_id,
426
+ random_seed=random_seed,
427
+ max_count=max_count,
428
+ )
429
+ if tpca_waves is None:
430
+ return None, None
431
+
432
+ waveforms = tpca_waves.waveforms
433
+
434
+ # remove chans with no signal at all
435
+ not_entirely_nan_channels = np.flatnonzero(
436
+ np.isfinite(waveforms[:, 0]).any(axis=0)
437
+ )
438
+ if (
439
+ not_entirely_nan_channels.size
440
+ and not_entirely_nan_channels.size < waveforms.shape[2]
441
+ ):
442
+ waveforms = waveforms[:, :, not_entirely_nan_channels]
443
+
444
+ waveforms = waveforms.reshape(len(waveforms), -1)
445
+ no_nan = np.flatnonzero(~np.isnan(waveforms).any(axis=1))
446
+
447
+ features = np.full((len(waveforms), rank), np.nan, dtype=waveforms.dtype)
448
+ if no_nan.size < rank:
449
+ return tpca_waves.which, features
450
+
451
+ pca = PCA(rank, random_state=random_seed, whiten=True)
452
+ if no_nan.size > max_wfs_fit:
453
+ rg = np.random.default_rng(random_state)
454
+ choices = rg.choice(no_nan, size=max_wfs_fit, replace=False)
455
+ choices.sort()
456
+ pca.fit(waveforms[choices])
457
+ # features[no_nan] = pca.transform(waveforms[no_nan])
458
+ else:
459
+ # features[no_nan] = pca.fit_transform(waveforms[no_nan])
460
+ pca.fit(waveforms[no_nan])
461
+ features = pca.transform(
462
+ np.where(np.isfinite(waveforms), waveforms, pca.mean_[None])
463
+ )
464
+ return tpca_waves.which, features
465
+
466
+ def unit_max_channel(self, unit_id) -> int:
467
+ assert self.coarse_template_data is not None
468
+ temp = self.coarse_template_data.unit_templates(unit_id)
469
+ assert temp.ndim == 3 and temp.shape[0] == np.atleast_1d(unit_id).size
470
+
471
+ which = self.in_unit(unit_id)
472
+ if self.motion_est is not None:
473
+ assert self.z is not None
474
+ assert self.registered_z is not None
475
+ times_seconds = getattr(self.sorting, "times_seconds", None)
476
+ assert times_seconds is not None
477
+ n_pitches_shift = get_spike_pitch_shifts(
478
+ self.z[which],
479
+ geom=self.geom,
480
+ registered_depths_um=self.registered_z[which],
481
+ times_s=times_seconds[which],
482
+ motion_est=self.motion_est,
483
+ )
484
+ covered_chans = get_stable_channels(
485
+ geom=self.geom,
486
+ registered_geom=self.registered_geom,
487
+ channels=self.sorting.channels[which],
488
+ channel_index=self.vis_channel_index,
489
+ n_pitches_shift=n_pitches_shift,
490
+ )[0]
491
+ else:
492
+ covered_chans = self.vis_channel_index[self.sorting.channels[which]]
493
+ covered_chans = np.unique(covered_chans)
494
+ covered_chans = covered_chans[covered_chans < len(self.vis_channel_index)]
495
+ amp_vec = np.ptp(temp.mean(0), 0)
496
+ max_chan = covered_chans[amp_vec[covered_chans].argmax()]
497
+ return max_chan.item()
498
+
499
+ def unit_select_channels(
500
+ self,
501
+ unit_id,
502
+ which,
503
+ waveforms,
504
+ read_chans=None,
505
+ main_channel=None,
506
+ ):
507
+ if read_chans is None:
508
+ read_chans = self.sorting.channels[which]
509
+ if main_channel is None:
510
+ main_channel = self.unit_max_channel(unit_id)
511
+
512
+ show_chans = self.vis_channel_index[main_channel]
513
+ show_valid = show_chans < len(self.registered_geom)
514
+ show_chans = show_chans[show_valid]
515
+
516
+ if self.shifting:
517
+ assert self.z is not None
518
+ assert self.registered_z is not None
519
+ times_seconds = getattr(self.sorting, "times_seconds", None)
520
+ assert times_seconds is not None
521
+ n_pitches_shift = get_spike_pitch_shifts(
522
+ self.z[which],
523
+ geom=self.geom,
524
+ registered_depths_um=self.registered_z[which],
525
+ times_s=times_seconds[which],
526
+ motion_est=self.motion_est,
527
+ )
528
+ else:
529
+ n_pitches_shift = None
530
+
531
+ if self.extract_channel_index is None:
532
+ read_channel_index = self.vis_channel_index
533
+ else:
534
+ read_channel_index = self.extract_channel_index
535
+
536
+ waveforms_valid = get_waveforms_on_static_channels(
537
+ waveforms=waveforms,
538
+ geom=self.geom,
539
+ n_pitches_shift=n_pitches_shift,
540
+ main_channels=read_chans,
541
+ channel_index=read_channel_index,
542
+ target_channels=show_chans,
543
+ registered_geom=self.registered_geom,
544
+ )
545
+ waveforms = np.full_like(
546
+ waveforms_valid,
547
+ shape=(*waveforms_valid.shape[:2], show_valid.size),
548
+ fill_value=np.nan,
549
+ )
550
+ waveforms[:, :, show_valid] = waveforms_valid
551
+
552
+ return waveforms, main_channel
553
+
554
+ def nearby_coarse_templates(self, unit_id, n_neighbors=5):
555
+ td = self.coarse_template_data
556
+ assert td is not None
557
+ assert self.merge_distances is not None
558
+
559
+ unit_ix = np.searchsorted(td.unit_ids, unit_id)
560
+ unit_dists = self.merge_distances[unit_ix]
561
+ distance_order = np.argsort(unit_dists)
562
+ distance_order = np.concatenate(
563
+ ([unit_ix], distance_order[distance_order != unit_ix])
564
+ )
565
+ # assert distance_order[0] == unit_ix
566
+ neighb_ixs = distance_order[:n_neighbors]
567
+ neighb_ids = td.unit_ids[neighb_ixs]
568
+ neighb_dists = self.merge_distances[neighb_ixs[:, None], neighb_ixs[None, :]]
569
+ neighb_coarse_templates = td.templates[neighb_ixs]
570
+ return neighb_ixs, neighb_ids, neighb_dists, neighb_coarse_templates
571
+
572
+ def spike_isis(self):
573
+ return sorting_isis(self.sorting)
574
+
575
+ def viol_rate(self, dt_ms=0.8):
576
+ viol_rates = np.zeros(self.unit_ids.shape)
577
+ for j, uid in enumerate(self.unit_ids):
578
+ inu = self.in_unit(uid)
579
+ if not inu.size > 1:
580
+ continue
581
+ t_ms = self.sorting.times_seconds[inu] * 1000 # type: ignore
582
+ isi = np.diff(t_ms)
583
+ viol_rates[j] = (np.abs(isi) < dt_ms).mean()
584
+ return viol_rates
585
+
586
+
587
+ @databag
588
+ class WaveformsBag:
589
+ temporal_slice: slice | None
590
+ which: np.ndarray
591
+ waveforms: np.ndarray
592
+ main_channel: int
593
+ geom: np.ndarray
594
+ channel_index: np.ndarray