DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1128 @@
1
+ from collections import namedtuple
2
+ from copy import copy
3
+ from dataclasses import replace
4
+ from pathlib import Path
5
+ from typing import Generator, Sequence, cast, Self, Literal
6
+ import warnings
7
+
8
+ import h5py
9
+ import numpy as np
10
+ import torch
11
+ from spikeinterface.core import NumpySorting, get_random_data_chunks
12
+ from tqdm.auto import tqdm
13
+
14
+ from ..detect import detect_and_deduplicate
15
+ from ..util.logging_util import get_logger
16
+ from ..util.py_util import resolve_path
17
+ from .waveform_util import make_channel_index
18
+
19
+ logger = get_logger(__name__)
20
+
21
+ # this is a data type used in the peeling code to store info about
22
+ # the datasets which are being computed
23
+ # the featurizers in transform have a .spike_dataset property which
24
+ # is this type
25
+ SpikeDataset = namedtuple("SpikeDataset", ["name", "shape_per_spike", "dtype"])
26
+
27
+
28
+ class DARTsortSorting:
29
+ """Class which holds spike times, channels, and labels
30
+
31
+ This class holds our algorithm state.
32
+ Initially the sorter doesn't have unit labels, so these are optional.
33
+ Export me to a SpikeInterface NumpySorting with .to_numpy_sorting()
34
+
35
+ When you instantiate this with from_peeling_hdf5, if the
36
+ flag load_simple_features is True (default), then additional
37
+ features of spikes will be loaded into memory -- like localizations,
38
+ which you can access like `sorting.point_source_localizations[...]`.
39
+ """
40
+
41
+ def __init__(
42
+ self,
43
+ *,
44
+ times_samples: np.ndarray,
45
+ channels: np.ndarray,
46
+ labels: np.ndarray | None,
47
+ parent_h5_path: str | Path | None = None,
48
+ sampling_frequency: float | int = 30000.0,
49
+ persistent_features: dict[str, np.ndarray] | None = None,
50
+ ephemeral_features: dict[str, np.ndarray] | None = None,
51
+ ):
52
+ self.n_spikes = times_samples.shape[0]
53
+ if parent_h5_path is not None:
54
+ parent_h5_path = resolve_path(parent_h5_path)
55
+ self.parent_h5_path = parent_h5_path
56
+ self.sampling_frequency = float(sampling_frequency)
57
+
58
+ assert times_samples.dtype.kind == "i"
59
+ self.times_samples = times_samples
60
+ assert channels.dtype.kind == "i"
61
+ assert channels.shape == (self.n_spikes,)
62
+ self.channels = channels
63
+ if labels is not None:
64
+ assert labels.shape == (self.n_spikes,)
65
+ assert labels.dtype.kind == "i"
66
+ self.labels = labels
67
+
68
+ self._ephemeral_feature_names = []
69
+ if ephemeral_features is not None:
70
+ for k, v in ephemeral_features.items():
71
+ check_shape = not self._no_check_needed(k)
72
+ self.add_ephemeral_feature(k, v, check_shape=check_shape)
73
+
74
+ self._loaded_persistent_features = []
75
+ if persistent_features is not None:
76
+ for k, v in persistent_features.items():
77
+ check_shape = not self._no_check_needed(k)
78
+ self._register_persistent_feature(k, v, check_shape=check_shape)
79
+
80
+ @property
81
+ def unit_ids(self) -> np.ndarray:
82
+ if self.labels is None:
83
+ return np.array([], dtype=np.int64)
84
+ u = np.unique(self.labels)
85
+ return u[u >= 0]
86
+
87
+ @property
88
+ def n_units(self) -> int:
89
+ return self.unit_ids.shape[0]
90
+
91
+ def copy(self) -> Self:
92
+ """Deep-enough copy."""
93
+ other = copy(self)
94
+ other._ephemeral_feature_names = self._ephemeral_feature_names.copy()
95
+ other._loaded_persistent_features = self._loaded_persistent_features.copy()
96
+ return other
97
+
98
+ def ephemeral_replace(
99
+ self, *, check_shapes=True, **new_features: np.ndarray
100
+ ) -> Self:
101
+ other = self.copy()
102
+ for k, v in new_features.items():
103
+ if k in ("times_samples", "channels", "labels"):
104
+ if check_shapes:
105
+ other._check_shape(k, v)
106
+ setattr(other, k, v)
107
+ else:
108
+ check = check_shapes and not self._no_check_needed(k)
109
+ other.add_ephemeral_feature(k, v, check_shape=check, overwrite=True)
110
+ return other
111
+
112
+ def has_persistent_labels(self) -> bool:
113
+ """Are my .labels those from the hdf5 file?"""
114
+ if self.parent_h5_path is None:
115
+ return False
116
+ if self.labels is None:
117
+ return False
118
+ if "labels" in self._ephemeral_feature_names:
119
+ assert "labels" not in self._loaded_persistent_features
120
+ return False
121
+ try:
122
+ return np.array_equal(self.labels, self._load_dataset("labels"))
123
+ except KeyError:
124
+ return False
125
+
126
+ # interface for setting features
127
+
128
+ def add_ephemeral_feature(
129
+ self,
130
+ feature_name: str,
131
+ feature: np.ndarray,
132
+ check_shape: bool | None = None,
133
+ overwrite=False,
134
+ ):
135
+ """
136
+ Ephemeral features are accessible as properties and persisted to/from .npz,
137
+ but not saved in the .h5.
138
+ """
139
+ if check_shape is None:
140
+ check_shape = not self._no_check_needed(feature_name)
141
+ if check_shape:
142
+ self._check_shape(feature_name, feature)
143
+
144
+ already_ephemeral = feature_name in self._ephemeral_feature_names
145
+ already_attr = hasattr(self, feature_name)
146
+ if already_ephemeral:
147
+ assert already_attr
148
+ if already_attr and not overwrite:
149
+ raise ValueError(
150
+ f"Can't add feature {feature_name}, since it already exists."
151
+ )
152
+ if not already_ephemeral:
153
+ self._ephemeral_feature_names.append(feature_name)
154
+ setattr(self, feature_name, feature)
155
+
156
+ def remove_ephemeral_feature(self, feature_name: str):
157
+ assert feature_name in self._ephemeral_feature_names
158
+ assert hasattr(self, feature_name)
159
+ self._ephemeral_feature_names = [
160
+ k for k in self._ephemeral_feature_names if k != feature_name
161
+ ]
162
+ delattr(self, feature_name)
163
+
164
+ def unload_persistent_feature(self, feature_name: str):
165
+ assert feature_name in self._loaded_persistent_features
166
+ assert hasattr(self, feature_name)
167
+ self._loaded_persistent_features = [
168
+ k for k in self._loaded_persistent_features if k != feature_name
169
+ ]
170
+ delattr(self, feature_name)
171
+
172
+ def add_feature(self, feature_name: str, feature: np.ndarray, check_shape=True):
173
+ """Try to save a feature to h5, else register as ephemeral."""
174
+ if self.parent_h5_path is None:
175
+ self.add_ephemeral_feature(feature_name, feature, check_shape)
176
+ else:
177
+ self._register_persistent_feature(feature_name, feature, check_shape)
178
+
179
+ def remove_feature(self, feature_name: str):
180
+ if feature_name in self._loaded_persistent_features:
181
+ self.unload_persistent_feature(feature_name)
182
+ elif feature_name in self._ephemeral_feature_names:
183
+ self.remove_ephemeral_feature(feature_name)
184
+ else:
185
+ raise ValueError(f"Sorting doesn't have {feature_name}.")
186
+
187
+ def _register_persistent_feature(
188
+ self, feature_name: str, feature: np.ndarray, check_shape=True
189
+ ):
190
+ """Persistent features are written to the h5."""
191
+ if self.parent_h5_path is None:
192
+ raise ValueError(
193
+ f"Can't register persistent feature {feature_name}, because "
194
+ f"there is no .hdf5 file.",
195
+ )
196
+ if check_shape:
197
+ self._check_shape(feature_name, feature)
198
+ self._loaded_persistent_features.append(feature_name)
199
+ setattr(self, feature_name, feature)
200
+ try:
201
+ with h5py.File(
202
+ self.parent_h5_path, "r", libver="latest", locking=False
203
+ ) as h5:
204
+ if feature_name not in h5:
205
+ logger.dartsortdebug(
206
+ "Registering persistent feature %s to %s.",
207
+ feature_name,
208
+ self.parent_h5_path,
209
+ )
210
+ h5.create_dataset(feature_name, data=feature)
211
+ except FileNotFoundError:
212
+ logger.warning(
213
+ f"Sorting's parent h5 file {self.parent_h5_path} is gone when registering "
214
+ f"persistent feature {feature_name}. Will continue, but this sorting won't "
215
+ "persist correctly.",
216
+ stacklevel=3,
217
+ )
218
+
219
+ # save / load
220
+
221
+ @classmethod
222
+ def from_peeling_hdf5(
223
+ cls,
224
+ h5_path: str | Path,
225
+ *,
226
+ times_samples_dataset="times_samples",
227
+ channels_dataset="channels",
228
+ labels_dataset="labels",
229
+ load_feature_names: Sequence[str] | None = None,
230
+ load_simple_features=True,
231
+ load_all_features=False,
232
+ ) -> Self:
233
+ """Load sorting from .hdf5 format saved by peelers
234
+
235
+ Arguments
236
+ ---------
237
+ load_feature_names : optional list of str
238
+ Load exactly these features, plus geom/channel index.
239
+ load_simple_features : bool
240
+ If load_feature_names unspecified, load all scalar or vector
241
+ features (per spike), but no matrix-valued features like waveforms
242
+ or multi-channel PCA features.
243
+ load_all_features : bool
244
+ """
245
+ h5_path = resolve_path(h5_path, strict=True)
246
+
247
+ with h5py.File(h5_path, "r", libver="latest", locking=False) as h5:
248
+ times_samples = cast(h5py.Dataset, h5[times_samples_dataset])[:]
249
+ n = times_samples.shape[0]
250
+ channels = cast(h5py.Dataset, h5[channels_dataset])[:]
251
+ sampling_frequency = float(
252
+ cast(h5py.Dataset, h5["sampling_frequency"])[()].item()
253
+ )
254
+ if labels_dataset in h5:
255
+ labels = cast(h5py.Dataset, h5[labels_dataset])[:]
256
+ else:
257
+ labels = None
258
+
259
+ already_loaded = [
260
+ times_samples_dataset,
261
+ channels_dataset,
262
+ labels_dataset,
263
+ "sampling_frequency",
264
+ ]
265
+ if load_feature_names is None and load_all_features:
266
+ load_feature_names = list(h5.keys())
267
+ elif load_feature_names is None and load_simple_features:
268
+ load_feature_names = []
269
+ for k in h5.keys():
270
+ if k in already_loaded:
271
+ continue
272
+ if cls._no_check_needed(k):
273
+ load_feature_names.append(k)
274
+ continue
275
+ dset = cast(h5py.Dataset, h5[k])
276
+ is_simple = 1 <= dset.ndim <= 2 and dset.shape[0] == n
277
+ if is_simple:
278
+ load_feature_names.append(k)
279
+ elif load_feature_names is None:
280
+ load_feature_names = [k for k in h5.keys() if cls._no_check_needed(k)]
281
+ assert load_feature_names is not None
282
+ load_feature_names = [
283
+ k for k in load_feature_names if k not in already_loaded
284
+ ]
285
+ persistent_features = {
286
+ k: cast(h5py.Dataset, h5[k])[:] for k in load_feature_names
287
+ }
288
+
289
+ return cls(
290
+ times_samples=times_samples,
291
+ channels=channels,
292
+ labels=labels,
293
+ sampling_frequency=sampling_frequency,
294
+ persistent_features=persistent_features,
295
+ parent_h5_path=h5_path,
296
+ )
297
+
298
+ def save(self, sorting_npz: str | Path):
299
+ """Support persisting myself in non-h5-supportable cases
300
+
301
+ Cases:
302
+ - When there is no h5!
303
+ - When my labels disagree with the .h5 labels.
304
+
305
+ This is done by saving to .npz, with a pointer to the .h5 file if
306
+ it exists.
307
+ """
308
+ sorting_npz = resolve_path(sorting_npz)
309
+ data = dict(
310
+ times_samples=self.times_samples,
311
+ channels=self.channels,
312
+ sampling_frequency=np.array(self.sampling_frequency),
313
+ )
314
+ if self.labels is not None:
315
+ data["labels"] = self.labels
316
+
317
+ have_hdf5 = self.parent_h5_path is not None
318
+ if have_hdf5:
319
+ # path needs to be relative to npz path's parent in case user moves stuff
320
+ h5p = resolve_path(self.parent_h5_path, strict=True)
321
+ h5p = h5p.relative_to(sorting_npz.parent)
322
+ data["parent_h5_path"] = np.array(str(h5p))
323
+ for k in self._ephemeral_feature_names:
324
+ data[k] = getattr(self, k)
325
+ data["ephemeral_feature_names"] = np.array(self._ephemeral_feature_names)
326
+ data["loaded_persistent_features"] = np.array(self._loaded_persistent_features)
327
+ np.savez(sorting_npz, **data, allow_pickle=False)
328
+
329
+ @classmethod
330
+ def load(cls, sorting_npz, additional_persistent_features=None) -> Self:
331
+ sorting_npz = resolve_path(sorting_npz, strict=True)
332
+ with np.load(sorting_npz) as data:
333
+ times_samples = data["times_samples"]
334
+ channels = data["channels"]
335
+ labels = data.get("labels", None)
336
+ sampling_frequency = data["sampling_frequency"]
337
+ if isinstance(sampling_frequency, np.ndarray):
338
+ sampling_frequency = sampling_frequency.item()
339
+ parent_h5_path = data.get("parent_h5_path", None)
340
+ if "ephemeral_feature_names" in data:
341
+ ephemeral_features = {
342
+ k: data[k] for k in data["ephemeral_feature_names"]
343
+ }
344
+ else:
345
+ ephemeral_features = {}
346
+ loaded_persistent_features = data.get("loaded_persistent_features", [])
347
+
348
+ if parent_h5_path is not None:
349
+ parent_h5_path = parent_h5_path.item()
350
+ assert isinstance(parent_h5_path, str)
351
+ parent_h5_path = sorting_npz.parent / Path(parent_h5_path)
352
+ parent_h5_path = resolve_path(parent_h5_path, strict=True)
353
+ if additional_persistent_features:
354
+ loaded_persistent_features = set(
355
+ loaded_persistent_features + additional_persistent_features
356
+ )
357
+ self = cls.from_peeling_hdf5(
358
+ parent_h5_path,
359
+ load_feature_names=list(loaded_persistent_features),
360
+ )
361
+ if labels is not None:
362
+ ephemeral_features["labels"] = labels
363
+ return self.ephemeral_replace(
364
+ times_samples=times_samples, channels=channels, **ephemeral_features
365
+ )
366
+ assert not loaded_persistent_features
367
+
368
+ return cls(
369
+ times_samples=times_samples,
370
+ channels=channels,
371
+ labels=labels,
372
+ sampling_frequency=sampling_frequency,
373
+ parent_h5_path=parent_h5_path,
374
+ ephemeral_features=ephemeral_features,
375
+ )
376
+
377
+ def to_numpy_sorting(self) -> NumpySorting:
378
+ return NumpySorting.from_samples_and_labels(
379
+ samples_list=self.times_samples,
380
+ labels_list=self.labels,
381
+ sampling_frequency=self.sampling_frequency,
382
+ )
383
+
384
+ def mask(self, mask: np.ndarray) -> Self:
385
+ assert mask.ndim == 1
386
+ if np.dtype(mask.dtype).kind == "b":
387
+ assert mask.shape == (self.n_spikes,)
388
+ mask = np.flatnonzero(mask)
389
+ assert mask.max() < self.n_spikes
390
+
391
+ if self.labels is None:
392
+ labels = None
393
+ else:
394
+ labels = self.labels[mask]
395
+
396
+ eph = {}
397
+ for k in self._ephemeral_feature_names:
398
+ assert k != "mask_indices" # no recursion...
399
+ v = getattr(self, k)
400
+ if self._no_check_needed(k):
401
+ eph[k] = v
402
+ else:
403
+ eph[k] = v[mask]
404
+ eph["mask_indices"] = mask
405
+
406
+ per = {}
407
+ for k in self._loaded_persistent_features:
408
+ assert k != "mask_indices" # no recursion...
409
+ v = getattr(self, k)
410
+ if self._no_check_needed(k):
411
+ per[k] = v
412
+ else:
413
+ per[k] = v[mask]
414
+
415
+ return self.__class__(
416
+ times_samples=self.times_samples[mask],
417
+ channels=self.channels[mask],
418
+ labels=labels,
419
+ parent_h5_path=self.parent_h5_path,
420
+ sampling_frequency=self.sampling_frequency,
421
+ persistent_features=per,
422
+ ephemeral_features=eph,
423
+ )
424
+
425
+ def drop_missing(self) -> Self:
426
+ assert self.labels is not None
427
+ return self.mask(self.labels >= 0)
428
+
429
+ def drop_doubles(self):
430
+ assert self.labels is not None
431
+ viol_ixs = []
432
+ for uid in self.unit_ids:
433
+ inu = np.flatnonzero(self.labels == uid)
434
+ tu = self.times_samples[inu]
435
+ # this has the first indices of each pair. +1 has the second indices.
436
+ viol = np.flatnonzero(np.diff(tu) == 0)
437
+ viol_ixs.append(viol + 1)
438
+ viol_ixs = np.concatenate(viol_ixs)
439
+ if viol_ixs.size:
440
+ logger.dartsortdebug(f"Dropping {viol_ixs.size} duplicates.")
441
+ labels = self.labels.copy()
442
+ labels[viol_ixs] = -1
443
+ return self.ephemeral_replace(labels=labels)
444
+ else:
445
+ return self
446
+
447
+ def flatten(self) -> Self:
448
+ assert self.labels is not None
449
+ valid = np.flatnonzero(self.labels >= 0)
450
+ _, flat_labels = np.unique(self.labels[valid], return_inverse=True)
451
+ new_labels = np.full_like(self.labels, -1)
452
+ new_labels[valid] = flat_labels
453
+ return self.ephemeral_replace(labels=new_labels)
454
+
455
+ def __str__(self):
456
+ name = self.__class__.__name__
457
+ ns = self.n_spikes
458
+ nu = self.n_units
459
+ unit_str = f"{nu} unit" + "s" * (nu > 1)
460
+ feat_str = " "
461
+ if self._loaded_persistent_features:
462
+ s = ", ".join(self._loaded_persistent_features)
463
+ feat_str += f"Loaded HDF5 features: {s}. "
464
+ if self._ephemeral_feature_names:
465
+ s = ", ".join(self._ephemeral_feature_names)
466
+ feat_str += f"Features: {s}. "
467
+ h5_str = ""
468
+ if self.parent_h5_path:
469
+ h5_str = f"From HDF5 file {self.parent_h5_path}."
470
+ if self.labels is not None:
471
+ noise_prop = (self.labels < 0).mean().item()
472
+ noise_pct = 100 * noise_prop
473
+ noise_str = f" ({noise_pct:.2f}% noise)"
474
+ else:
475
+ noise_str = ""
476
+ return f"{name}: {ns} spikes{noise_str}, {unit_str}.{feat_str}{h5_str}"
477
+
478
+ def __repr__(self):
479
+ return str(self)
480
+
481
+ def __len__(self):
482
+ return self.n_spikes
483
+
484
+ @staticmethod
485
+ def _is_geom_related(k) -> bool:
486
+ return k == "geom" or k.endswith("channel_index")
487
+
488
+ @staticmethod
489
+ def _is_unit_related(k) -> bool:
490
+ return k.startswith("unit")
491
+
492
+ @classmethod
493
+ def _no_check_needed(cls, k) -> bool:
494
+ return cls._is_geom_related(k) or cls._is_unit_related(k)
495
+
496
+ def _check_shape(self, feature_name: str, feature: np.ndarray):
497
+ if feature.shape[0] != self.n_spikes:
498
+ raise ValueError(
499
+ f"Feature {feature_name}'s shape {feature.shape} didn't agree with spike count {self.n_spikes}."
500
+ )
501
+
502
+ def _load_dataset(self, dataset_name: str) -> np.ndarray:
503
+ assert self.parent_h5_path is not None
504
+ with h5py.File(self.parent_h5_path, "r", locking=False) as h5:
505
+ dset = h5[dataset_name]
506
+ assert isinstance(dset, h5py.Dataset)
507
+ return dset[:]
508
+
509
+ def slice_feature_by_name(
510
+ self, dataset_name: str, mask: np.ndarray | slice = slice(None)
511
+ ) -> np.ndarray:
512
+ if hasattr(self, dataset_name):
513
+ return getattr(self, dataset_name)[mask]
514
+
515
+ # otherwise, we don't have it loaded
516
+ assert dataset_name not in self._ephemeral_feature_names
517
+ assert dataset_name not in self._loaded_persistent_features
518
+
519
+ # but we can try to load it
520
+ if self.parent_h5_path is None:
521
+ raise ValueError(f"Can't load feature {dataset_name} with no HDF5.")
522
+ if isinstance(mask, slice) and mask == slice(None):
523
+ return self._load_dataset(dataset_name)
524
+
525
+ # h5 direct read is fine for a few indices
526
+ if isinstance(mask, np.ndarray) and mask.dtype.kind != "b":
527
+ if mask.size <= 768:
528
+ with h5py.File(self.parent_h5_path, "r", locking=False) as h5:
529
+ return cast(h5py.Dataset, h5[dataset_name])[mask]
530
+
531
+ # mask needs to be boolean for _read_by_chunk
532
+ if not isinstance(mask, np.ndarray) or mask.dtype.kind != "b":
533
+ h5_mask = np.zeros(self.n_spikes, dtype=np.bool_)
534
+ h5_mask[mask] = True
535
+ else:
536
+ h5_mask = mask
537
+
538
+ with h5py.File(self.parent_h5_path, "r", locking=False) as h5:
539
+ dset = h5[dataset_name]
540
+ assert isinstance(dset, h5py.Dataset)
541
+ assert dset.shape[0] == self.n_spikes == h5_mask.shape[0]
542
+ return _read_by_chunk(mask, dset, show_progress=False)
543
+
544
+
545
+ def load_h5(f: str | Path) -> DARTsortSorting:
546
+ return DARTsortSorting.from_peeling_hdf5(h5_path=f)
547
+
548
+
549
+ def try_get_model_dir(sorting: DARTsortSorting) -> Path | None:
550
+ if sorting.parent_h5_path is None:
551
+ return None
552
+ h5_path = resolve_path(sorting.parent_h5_path)
553
+ model_dir = h5_path.parent / f"{h5_path.stem}_models"
554
+ if model_dir.exists():
555
+ assert model_dir.is_dir()
556
+ return model_dir
557
+ else:
558
+ return None
559
+
560
+
561
+ def get_featurization_pipeline(sorting, featurization_pipeline_pt=None):
562
+ """Look for the pipeline in the usual place."""
563
+ from dartsort.transform import WaveformPipeline
564
+
565
+ if isinstance(sorting, Path):
566
+ base_dir = sorting.parent
567
+ stem = sorting.stem
568
+ # TODO how to type this better... don't understand.
569
+ geom = channel_index = None # type: ignore
570
+ else:
571
+ assert isinstance(sorting, DARTsortSorting)
572
+ if sorting.parent_h5_path is None:
573
+ raise ValueError("Can't load featurization pipeline.")
574
+
575
+ h5_path = resolve_path(sorting.parent_h5_path)
576
+ base_dir = h5_path.parent
577
+ stem = h5_path.stem
578
+
579
+ geom = getattr(sorting, "geom", None) # type: ignore
580
+ channel_index = getattr(sorting, "channel_index", None) # type: ignore
581
+
582
+ model_dir = base_dir / f"{stem}_models"
583
+ if geom is None or channel_index is None:
584
+ with h5py.File(base_dir / f"{stem}.h5", "r", locking=False) as h5:
585
+ geom: np.ndarray = h5["geom"][:] # type: ignore
586
+ channel_index: np.ndarray = h5["channel_index"][:] # type: ignore
587
+ assert geom is not None
588
+ assert channel_index is not None
589
+
590
+ if featurization_pipeline_pt is None:
591
+ featurization_pipeline_pt = model_dir / "featurization_pipeline.pt"
592
+ if not featurization_pipeline_pt.exists():
593
+ raise ValueError(f"No file at {featurization_pipeline_pt=}")
594
+ pipeline = WaveformPipeline.from_state_dict_pt(
595
+ geom, channel_index, featurization_pipeline_pt
596
+ )
597
+ return pipeline, featurization_pipeline_pt
598
+
599
+
600
+ def get_tpca(sorting, tpca_name="collisioncleaned_tpca_features"):
601
+ """Look for the TemporalPCAFeaturizer in the usual place."""
602
+ pipeline, _ = get_featurization_pipeline(sorting)
603
+ return pipeline.get_transformer(tpca_name)
604
+
605
+
606
+ def load_stored_tsvd(sorting, tsvd_name="collisioncleaned_basis", to_sklearn=True):
607
+ from ..transform import BaseTemporalPCA
608
+
609
+ if sorting.parent_h5_path is None:
610
+ logger.info("Couldn't load stored basis.")
611
+ return None
612
+ pipeline, pt_path = get_featurization_pipeline(sorting)
613
+ tsvd = pipeline.get_transformer(tsvd_name)
614
+ assert tsvd is not None
615
+ assert tsvd.name == tsvd_name
616
+ assert isinstance(tsvd, BaseTemporalPCA)
617
+ if to_sklearn:
618
+ tsvd = tsvd.to_sklearn()
619
+ logger.info(
620
+ f"Loaded stored basis from %s (%s; components shape: %s).",
621
+ pt_path,
622
+ tsvd_name,
623
+ tsvd.components_.shape,
624
+ )
625
+ return tsvd
626
+
627
+
628
+ def get_labels(h5_path) -> np.ndarray:
629
+ with h5py.File(h5_path, "r") as h5:
630
+ return h5["labels"][:] # type: ignore
631
+
632
+
633
+ def get_residual_snips(h5_path) -> np.ndarray:
634
+ with h5py.File(h5_path, "r", locking=False) as h5:
635
+ return h5["residual"][:] # type: ignore
636
+
637
+
638
+ def sorting_isis(sorting: DARTsortSorting):
639
+ assert sorting.labels is not None
640
+ isis_ms = np.zeros(len(sorting))
641
+ for uid in sorting.unit_ids:
642
+ inu = np.flatnonzero(sorting.labels == uid)
643
+ t_ms = sorting.times_seconds[inu] * 1000 # type: ignore
644
+ isi = np.diff(t_ms)
645
+ isi = np.concatenate([[np.inf], np.abs(isi), [np.inf]])
646
+ isi = np.minimum(isi[1:], isi[:-1])
647
+ isis_ms[inu] = isi
648
+ return isis_ms
649
+
650
+
651
+ def keep_only_most_recent_spikes(
652
+ sorting,
653
+ n_min_spikes=250,
654
+ latest_time_sample=90_000_000,
655
+ ):
656
+ """
657
+ This function selects the n_min_spikes before latest_time (or most recent after latest_time)
658
+ """
659
+ new_labels = np.full(sorting.labels.shape, -1)
660
+ units = np.unique(sorting.labels)
661
+ units = units[units > -1]
662
+ for k in units:
663
+ idx_k = np.flatnonzero(sorting.labels == k)
664
+ before_time = sorting.times_samples[idx_k] < latest_time_sample
665
+ if before_time.sum() <= n_min_spikes:
666
+ idx_k = idx_k[:n_min_spikes]
667
+ new_labels[idx_k] = k
668
+ else:
669
+ idx_k = idx_k[before_time][-n_min_spikes:]
670
+ new_labels[idx_k] = k
671
+ new_sorting = sorting.ephemeral_replace(labels=new_labels)
672
+ return new_sorting
673
+
674
+
675
+ def check_recording(
676
+ rec,
677
+ threshold=5,
678
+ dedup_spatial_radius=75,
679
+ expected_value_range=1e4,
680
+ expected_spikes_per_sec=10_000,
681
+ num_chunks_per_segment=5,
682
+ dtype=torch.float,
683
+ ):
684
+ """Sanity check spike detection rate and data range of input recording."""
685
+
686
+ # grab random traces from throughout rec
687
+ random_chunks = get_random_data_chunks(
688
+ rec,
689
+ num_chunks_per_segment=num_chunks_per_segment,
690
+ chunk_size=min(rec.get_num_samples(), int(rec.sampling_frequency)),
691
+ concatenated=False,
692
+ )
693
+ dedup_channel_index = None
694
+ if dedup_spatial_radius:
695
+ dedup_channel_index = make_channel_index(
696
+ rec.get_channel_locations(), dedup_spatial_radius
697
+ )
698
+
699
+ # run detection and compute spike detection rate and data range
700
+ spike_rates = []
701
+ for chunk in random_chunks:
702
+ dres = detect_and_deduplicate(
703
+ torch.tensor(chunk, dtype=dtype),
704
+ threshold=threshold,
705
+ peak_sign="both",
706
+ dedup_channel_index=torch.tensor(dedup_channel_index),
707
+ )
708
+ times = dres[0]
709
+ del dres
710
+ chunk_len_s = rec.sampling_frequency / chunk.shape[0]
711
+ spike_rates.append(times.shape[0] / chunk_len_s)
712
+
713
+ avg_detections_per_second = np.mean(spike_rates)
714
+ max_abs = np.max(random_chunks)
715
+
716
+ failed = False
717
+ if avg_detections_per_second > expected_spikes_per_sec:
718
+ warnings.warn(
719
+ f"Detected {avg_detections_per_second:0.1f} spikes/s, which is "
720
+ "large. You may want to check that your data has been preprocessed, "
721
+ "including standardization. If it seems right, then you may need to "
722
+ "shrink the chunk_length_samples parameters in the configuration if "
723
+ "you experience memory issues.",
724
+ RuntimeWarning,
725
+ )
726
+ failed = True
727
+
728
+ if max_abs > expected_value_range:
729
+ warnings.warn(
730
+ f"Recording values exceed |{expected_value_range}|. You may want to "
731
+ "check that your data has been preprocessed, including standardization.",
732
+ RuntimeWarning,
733
+ )
734
+ failed = True
735
+
736
+ return failed, avg_detections_per_second, max_abs
737
+
738
+
739
+ def subset_sorting_by_spike_count(sorting, min_spikes=0, max_spikes=np.inf):
740
+ if not min_spikes:
741
+ return sorting
742
+
743
+ units, counts = np.unique(sorting.labels, return_counts=True)
744
+ invalid = np.logical_or(counts < min_spikes, counts > max_spikes)
745
+ bad_units = units[invalid]
746
+
747
+ new_labels = np.where(np.isin(sorting.labels, bad_units), -1, sorting.labels)
748
+
749
+ return sorting.ephemeral_replace(labels=new_labels)
750
+
751
+
752
+ def subsample_to_max_count(
753
+ sorting, max_spikes=256, seed: int | np.random.Generator = 0, discard_triaged=False
754
+ ):
755
+ units, counts = np.unique(sorting.labels, return_counts=True)
756
+ if counts.max() <= max_spikes:
757
+ return sorting
758
+
759
+ rg = np.random.default_rng(seed)
760
+ new_labels = sorting.labels.copy()
761
+ for u in units[counts > max_spikes]:
762
+ in_u = np.flatnonzero(sorting.labels == u)
763
+ new_labels[in_u] = -1
764
+ in_u = rg.choice(in_u, size=max_spikes, replace=False)
765
+ new_labels[in_u] = u
766
+
767
+ return sorting.ephemeral_replace(labels=new_labels)
768
+
769
+
770
+ def restrict_to_valid_times(sorting, recording, waveform_cfg, pad=0):
771
+ trough = waveform_cfg.trough_offset_samples(recording.sampling_frequency)
772
+ total = waveform_cfg.spike_length_samples(recording.sampling_frequency)
773
+ t_min = trough + pad
774
+ t_max = recording.get_total_samples() - (total - trough) - pad
775
+ new_labels = sorting.labels.copy()
776
+ new_labels[sorting.times_samples < t_min] = -1
777
+ new_labels[sorting.times_samples > t_max] = -1
778
+ return sorting.ephemeral_replace(labels=new_labels)
779
+
780
+
781
+ def subset_sorting_by_time_samples(
782
+ sorting, start_sample=0, end_sample=np.inf, reference_to_start_sample=True
783
+ ):
784
+ new_times = sorting.times_samples.copy()
785
+ new_labels = sorting.labels.copy()
786
+
787
+ in_range = (new_times >= start_sample) & (new_times < end_sample)
788
+ new_labels[~in_range] = -1
789
+
790
+ if reference_to_start_sample:
791
+ new_times -= start_sample
792
+
793
+ return sorting.ephemeral_replace(labels=new_labels, times_samples=new_times)
794
+
795
+
796
+ def subset_sorting_by_time_seconds(sorting, t_start=0, t_end=np.inf):
797
+ new_labels = sorting.labels.copy()
798
+ t_s = sorting.times_seconds
799
+ in_range = t_s == t_s.clip(t_start, t_end)
800
+ new_labels[~in_range] = -1
801
+
802
+ return sorting.ephemeral_replace(labels=new_labels)
803
+
804
+
805
+ def time_chunk_sortings(
806
+ sorting, recording=None, chunk_length_samples=None, chunk_time_ranges_s=None
807
+ ):
808
+ if chunk_time_ranges_s is None:
809
+ chunk_time_ranges_s = chunk_time_ranges(recording, chunk_length_samples)
810
+ chunk_sortings = [
811
+ subset_sorting_by_time_seconds(sorting, *tt) for tt in chunk_time_ranges_s
812
+ ]
813
+ return chunk_time_ranges_s, chunk_sortings
814
+
815
+
816
+ def reindex_sorting_labels(sorting):
817
+ new_labels = sorting.labels.copy()
818
+ kept = np.flatnonzero(new_labels >= 0)
819
+ _, new_labels[kept] = np.unique(new_labels[kept], return_inverse=True)
820
+ return sorting.ephemeral_replace(labels=new_labels)
821
+
822
+
823
+ def combine_sortings(sortings, dodge=False):
824
+ labels = np.full_like(sortings[0].labels, -1)
825
+ times_samples = sortings[0].times_samples.copy()
826
+ assert all(s.labels.size == sortings[0].labels.size for s in sortings)
827
+
828
+ if dodge:
829
+ label_to_sorting_index = []
830
+ label_to_original_label = []
831
+ else:
832
+ label_to_sorting_index = None
833
+ label_to_original_label = None
834
+
835
+ next_label = 0
836
+ for j, sorting in enumerate(sortings):
837
+ kept = np.flatnonzero(sorting.labels >= 0)
838
+ assert np.all(labels[kept] < 0)
839
+ labels[kept] = sorting.labels[kept] + next_label
840
+ if dodge:
841
+ assert label_to_sorting_index is not None
842
+ assert label_to_original_label is not None
843
+ n_new_labels = 0
844
+ if kept.size:
845
+ n_new_labels = 1 + sorting.labels[kept].max()
846
+ next_label += n_new_labels
847
+ label_to_sorting_index.append(np.full(n_new_labels, j))
848
+ label_to_original_label.append(np.arange(n_new_labels))
849
+ times_samples[kept] = sorting.times_samples[kept]
850
+
851
+ sorting = sortings[0].ephemeral_replace(labels=labels, times_samples=times_samples)
852
+
853
+ if dodge:
854
+ assert label_to_sorting_index is not None
855
+ assert label_to_original_label is not None
856
+ label_to_sorting_index = np.concatenate(label_to_sorting_index)
857
+ label_to_original_label = np.concatenate(label_to_original_label)
858
+ return label_to_sorting_index, label_to_original_label, sorting
859
+ return sorting
860
+
861
+
862
+ # -- timing
863
+
864
+
865
+ def chunk_time_ranges(recording, chunk_length_samples=None):
866
+ if chunk_length_samples is None or chunk_length_samples == np.inf:
867
+ n_chunks = 1
868
+ else:
869
+ n_chunks = recording.get_num_samples() / chunk_length_samples
870
+ # we'll count the remainder as a chunk if it's at least 2/3 of one
871
+ n_chunks = np.floor(n_chunks) + (n_chunks - np.floor(n_chunks) > 0.66)
872
+ n_chunks = int(max(1, n_chunks))
873
+
874
+ # evenly divide the recording into chunks
875
+ assert recording.get_num_segments() == 1
876
+ start_time_s, end_time_s = recording._recording_segments[0].sample_index_to_time(
877
+ np.array([0, recording.get_num_samples() - 1])
878
+ )
879
+ chunk_times_s = np.linspace(start_time_s, end_time_s, num=n_chunks + 1)
880
+ chunk_time_ranges_s = list(zip(chunk_times_s[:-1], chunk_times_s[1:]))
881
+
882
+ return chunk_time_ranges_s
883
+
884
+
885
+ # -- hdf5 util
886
+
887
+
888
+ def batched_h5_read(dataset, indices=None, mask=None, show_progress=False):
889
+ if mask is None:
890
+ mask = np.zeros(len(dataset), dtype=bool)
891
+ mask[indices] = 1
892
+ return _read_by_chunk(mask, dataset, show_progress=show_progress)
893
+
894
+
895
+ def _read_by_chunk(mask, dataset, show_progress=True):
896
+ """
897
+ mask : boolean array of shape dataset.shape[:1]
898
+ dataset : chunked h5py.Dataset
899
+ """
900
+ out = np.empty((mask.sum(), *dataset.shape[1:]), dtype=dataset.dtype)
901
+ n = 0
902
+ for sli, dsli in yield_chunks(dataset, show_progress):
903
+ m = np.flatnonzero(mask[sli])
904
+ nm = m.size
905
+ if not nm:
906
+ continue
907
+ x = dsli[m]
908
+ out[n : n + nm] = x
909
+ n += nm
910
+ return out
911
+
912
+
913
+ def yield_chunks(
914
+ dataset, show_progress=True, desc_prefix=None, fallback_chunk_length=4096
915
+ ) -> Generator[tuple[slice, np.ndarray], None, None]:
916
+ """Iterate chunks of an h5py dataset
917
+
918
+ The dataset can either not be chunked or chunked only on the first axis.
919
+ """
920
+ if dataset.chunks is None:
921
+ chunks = (
922
+ slice(s, min(s + fallback_chunk_length, len(dataset)))
923
+ for s in range(0, len(dataset), fallback_chunk_length)
924
+ )
925
+ else:
926
+ for c, s in zip(dataset.chunks[1:], dataset.shape[1:]):
927
+ if c == s:
928
+ continue
929
+ raise ValueError(
930
+ f"Dataset {dataset} can only be chunked on the first axis. "
931
+ f"Found {dataset.chunks=} with {dataset.shape=}."
932
+ )
933
+
934
+ chunks = dataset.iter_chunks()
935
+ # throw away slices along other axes
936
+ chunks = (chunk[0] for chunk in chunks)
937
+
938
+ if show_progress:
939
+ desc = dataset.name
940
+ if desc_prefix:
941
+ desc = f"{desc_prefix} {desc}"
942
+ if dataset.chunks is None:
943
+ n_chunks = int(np.ceil(dataset.shape[0] / fallback_chunk_length))
944
+ else:
945
+ n_chunks = int(np.ceil(dataset.shape[0] / dataset.chunks[0]))
946
+ chunks = tqdm(chunks, total=n_chunks, desc=desc)
947
+
948
+ for sli in chunks:
949
+ yield sli, dataset[sli]
950
+
951
+
952
+ def yield_masked_chunks(mask, dataset, show_progress=True, desc_prefix=None):
953
+ offset = 0
954
+ for sli, data in yield_chunks(
955
+ dataset, show_progress=show_progress, desc_prefix=desc_prefix
956
+ ):
957
+ source_ixs = np.flatnonzero(mask[sli])
958
+ dest_ixs = slice(offset, offset + source_ixs.size)
959
+ yield dest_ixs, data[source_ixs]
960
+ offset += source_ixs.size
961
+
962
+
963
+ # -- residual
964
+
965
+
966
+ def extract_random_snips(
967
+ rg: int | np.random.Generator,
968
+ chunk: np.ndarray | torch.Tensor,
969
+ n: int,
970
+ sniplen: int,
971
+ ):
972
+ """Grab n (or as many as can fit) random non-overlapping snips from chunk."""
973
+ rg = np.random.default_rng(rg)
974
+
975
+ # we can extract at most this many snips
976
+ n = min(n, max(1, chunk.shape[0] // sniplen))
977
+
978
+ # how many samples will not be covered
979
+ empty_len = chunk.shape[0] - sniplen * n
980
+ assert empty_len >= 0
981
+
982
+ # sample points spaced by sniplen + deltaj, where sum of
983
+ # delta is <= empty_len. use an extra slack diff for the <.
984
+ if empty_len == 0:
985
+ delta = np.zeros((n + 1,), dtype=np.int64)
986
+ else:
987
+ delta = rg.integers(0, empty_len, size=n + 1)
988
+ delta.sort()
989
+ delta = np.diff(delta)[:n]
990
+ times = np.cumsum(delta + sniplen) - sniplen
991
+ tixs = times[:, None] + np.arange(sniplen)
992
+ return chunk[tixs], times
993
+
994
+
995
+ # -- dataset subsampling
996
+
997
+
998
+ def subsample_waveforms(
999
+ hdf5_filename: str | Path | None = None,
1000
+ fit_sampling: Literal["random", "amp_reweighted"] = "random",
1001
+ random_state: int | np.random.Generator = 0,
1002
+ n_waveforms_fit=10_000,
1003
+ voltages_dataset_name="collisioncleaned_voltages",
1004
+ waveforms_dataset_name="collisioncleaned_waveforms",
1005
+ fit_max_reweighting=4.0,
1006
+ log_voltages=True,
1007
+ subsample_by_weighting=False,
1008
+ fixed_property_keys=("channels",),
1009
+ replace=True,
1010
+ h5=None,
1011
+ device: torch.device | str = "cpu",
1012
+ ) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
1013
+ random_state = np.random.default_rng(random_state)
1014
+
1015
+ need_open = h5 is None
1016
+ if need_open and hdf5_filename is not None:
1017
+ hdf5_filename = resolve_path(hdf5_filename, strict=True)
1018
+ h5 = h5py.File(hdf5_filename)
1019
+ elif need_open:
1020
+ raise ValueError("Need h5 or hdf5_filename.")
1021
+
1022
+ try:
1023
+ channels: np.ndarray = h5["channels"][:] # type: ignore
1024
+ n_wf = channels.shape[0]
1025
+ if not n_wf:
1026
+ emptyi = torch.tensor([], dtype=torch.long)
1027
+ wfshape = h5[waveforms_dataset_name].shape # type: ignore
1028
+ emptywf = torch.zeros(wfshape)
1029
+ return emptywf, dict(channels=emptyi)
1030
+ weights = fit_reweighting(
1031
+ h5=h5,
1032
+ log_voltages=log_voltages,
1033
+ fit_sampling=fit_sampling,
1034
+ fit_max_reweighting=fit_max_reweighting,
1035
+ voltages_dataset_name=voltages_dataset_name,
1036
+ )
1037
+ if n_wf > n_waveforms_fit and not subsample_by_weighting:
1038
+ choices = random_state.choice(
1039
+ n_wf, p=weights, size=n_waveforms_fit, replace=replace
1040
+ )
1041
+ if not replace:
1042
+ choices.sort()
1043
+ waveforms = batched_h5_read(h5[waveforms_dataset_name], choices)
1044
+ fixed_properties = {k: h5[k][choices] for k in fixed_property_keys} # type: ignore
1045
+ else:
1046
+ uchoices, ichoices = np.unique(choices, return_inverse=True)
1047
+ waveforms = batched_h5_read(h5[waveforms_dataset_name], uchoices)[
1048
+ ichoices
1049
+ ]
1050
+ fixed_properties = {
1051
+ k: h5[k][uchoices][ichoices] # type: ignore
1052
+ for k in fixed_property_keys
1053
+ }
1054
+ else:
1055
+ waveforms: np.ndarray = h5[waveforms_dataset_name][:] # type: ignore
1056
+ fixed_properties = {k: h5[k][:] for k in fixed_property_keys} # type: ignore
1057
+ finally:
1058
+ if need_open:
1059
+ h5.close()
1060
+ del h5
1061
+
1062
+ device = torch.device(device)
1063
+ waveformsr = torch.as_tensor(waveforms, device=device)
1064
+ fixed_properties = {
1065
+ k: torch.as_tensor(v, device=device) for k, v in fixed_properties.items()
1066
+ }
1067
+ if subsample_by_weighting and weights is not None:
1068
+ fixed_properties["weights"] = torch.as_tensor(weights, device=device)
1069
+ elif subsample_by_weighting:
1070
+ fixed_properties["weights"] = torch.ones(waveforms.shape[0], device=device)
1071
+
1072
+ return waveformsr, fixed_properties
1073
+
1074
+
1075
+ def fit_reweighting(
1076
+ voltages: np.ndarray | None = None, # type: ignore
1077
+ h5=None,
1078
+ hdf5_path=None,
1079
+ log_voltages=True,
1080
+ fit_sampling: Literal["random", "amp_reweighted"] = "random",
1081
+ fit_max_reweighting=4.0,
1082
+ voltages_dataset_name="voltages",
1083
+ ):
1084
+ if fit_sampling == "random":
1085
+ return None
1086
+ assert fit_sampling == "amp_reweighted"
1087
+
1088
+ if voltages is None:
1089
+ if h5 is not None:
1090
+ voltages = h5[voltages_dataset_name][:]
1091
+ elif hdf5_path is not None:
1092
+ with h5py.File(hdf5_path) as h5:
1093
+ voltages: np.ndarray = h5[voltages_dataset_name][:] # type: ignore
1094
+ else:
1095
+ assert False
1096
+ assert isinstance(voltages, np.ndarray)
1097
+
1098
+ from ..clustering.density import get_smoothed_density
1099
+
1100
+ if torch.is_tensor(voltages):
1101
+ voltages = voltages.numpy(force=True)
1102
+ if log_voltages:
1103
+ sign = np.sign(voltages)
1104
+ voltages = sign * np.log(np.abs(voltages))
1105
+ voltages = np.nan_to_num(voltages)
1106
+ sigma = 1.06 * voltages.std() * np.power(len(voltages), -0.2)
1107
+ assert np.isfinite(sigma)
1108
+ dens = get_smoothed_density(voltages[:, None], sigma=sigma)
1109
+ assert isinstance(dens, np.ndarray)
1110
+ sample_p = dens.mean() / dens
1111
+ sample_p = sample_p.clip(1.0 / fit_max_reweighting, fit_max_reweighting)
1112
+ sample_p = sample_p.astype(np.float64) # ensure double before normalizing
1113
+ sample_p /= sample_p.sum()
1114
+ return sample_p
1115
+
1116
+
1117
+ def divide_randomly(n_things, n_bins, rg):
1118
+ things_per_bin = np.zeros(n_bins, dtype=np.int64)
1119
+ n_even_split = n_things // n_bins
1120
+ things_per_bin += n_even_split
1121
+ n_things_remaining = n_things - n_bins * n_even_split
1122
+ assert n_things_remaining >= 0
1123
+ if n_things_remaining:
1124
+ rg = np.random.default_rng(rg)
1125
+ choices = rg.choice(n_bins, size=n_things_remaining)
1126
+ np.add.at(things_per_bin, choices, 1)
1127
+ assert things_per_bin.sum() == n_things
1128
+ return things_per_bin