DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1000 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
import tempfile
|
|
3
|
+
from concurrent.futures import CancelledError
|
|
4
|
+
from contextlib import contextmanager
|
|
5
|
+
from itertools import repeat
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from sys import getrefcount
|
|
8
|
+
from threading import Lock, local
|
|
9
|
+
from types import MappingProxyType
|
|
10
|
+
from typing import Any, Literal, TypedDict, overload
|
|
11
|
+
|
|
12
|
+
import h5py
|
|
13
|
+
import numpy as np
|
|
14
|
+
import torch
|
|
15
|
+
from spikeinterface.core.recording_tools import get_chunk_with_margin
|
|
16
|
+
from sympy import divisors
|
|
17
|
+
from tqdm.auto import tqdm
|
|
18
|
+
|
|
19
|
+
from ..transform import WaveformPipeline
|
|
20
|
+
from ..util import job_util
|
|
21
|
+
from ..util.data_util import (
|
|
22
|
+
SpikeDataset,
|
|
23
|
+
divide_randomly,
|
|
24
|
+
extract_random_snips,
|
|
25
|
+
subsample_waveforms,
|
|
26
|
+
)
|
|
27
|
+
from ..util.logging_util import get_logger
|
|
28
|
+
from ..util.multiprocessing_util import pool_from_cfg
|
|
29
|
+
from ..util.py_util import delay_keyboard_interrupt
|
|
30
|
+
from ..util.torch_util import BModule
|
|
31
|
+
|
|
32
|
+
logger = get_logger(__name__)
|
|
33
|
+
_lock = Lock()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class BasePeeler(BModule):
|
|
37
|
+
"""Base class for peeling operations (subtraction, deconv, etc)
|
|
38
|
+
|
|
39
|
+
Subtraction, deconv, and other things like just grabbing waveforms
|
|
40
|
+
and featurizing a preexisting spike train (implemented with optional
|
|
41
|
+
additional peeling of preexisting templates in grab.py) share a lot
|
|
42
|
+
of logic for featurization and parallelization and data saving/loading.
|
|
43
|
+
This class handles all of that stuff so that it can be shared.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
peel_kind = ""
|
|
47
|
+
|
|
48
|
+
def __init__(
|
|
49
|
+
self,
|
|
50
|
+
recording,
|
|
51
|
+
channel_index,
|
|
52
|
+
featurization_pipeline: WaveformPipeline | None = None,
|
|
53
|
+
chunk_length_samples=30_000,
|
|
54
|
+
chunk_margin_samples=0,
|
|
55
|
+
n_seconds_fit=40,
|
|
56
|
+
max_waveforms_fit=50_000,
|
|
57
|
+
n_waveforms_fit=20_000,
|
|
58
|
+
fit_max_reweighting=4.0,
|
|
59
|
+
fit_sampling: Literal["random", "amp_reweighted"] = "random",
|
|
60
|
+
fit_subsampling_random_state: int | np.random.Generator = 0,
|
|
61
|
+
trough_offset_samples=42,
|
|
62
|
+
spike_length_samples=121,
|
|
63
|
+
fixed_property_keys=("channels",),
|
|
64
|
+
dtype=torch.float,
|
|
65
|
+
):
|
|
66
|
+
if recording.get_num_segments() > 1:
|
|
67
|
+
raise ValueError("Peeling does not yet support multi-segment recordings.")
|
|
68
|
+
super().__init__()
|
|
69
|
+
self.recording = recording
|
|
70
|
+
self.chunk_length_samples: int = chunk_length_samples
|
|
71
|
+
self.chunk_margin_samples: int = chunk_margin_samples
|
|
72
|
+
self.n_seconds_fit: int = n_seconds_fit
|
|
73
|
+
self.max_waveforms_fit: int = max_waveforms_fit
|
|
74
|
+
self.n_waveforms_fit: int = n_waveforms_fit
|
|
75
|
+
self.trough_offset_samples: int = trough_offset_samples
|
|
76
|
+
self.spike_length_samples: int = spike_length_samples
|
|
77
|
+
self.fit_subsampling_random_state: np.random.Generator = np.random.default_rng(
|
|
78
|
+
fit_subsampling_random_state
|
|
79
|
+
)
|
|
80
|
+
self.dtype: torch.dtype = dtype
|
|
81
|
+
self.np_dtype = torch.empty((), dtype=dtype).numpy().dtype
|
|
82
|
+
if channel_index is not None:
|
|
83
|
+
self.register_buffer("channel_index", channel_index)
|
|
84
|
+
assert recording.get_num_channels() == channel_index.shape[0]
|
|
85
|
+
self.fit_sampling: Literal["random", "amp_reweighted"] = fit_sampling
|
|
86
|
+
self.fit_max_reweighting: float = fit_max_reweighting
|
|
87
|
+
self.featurization_pipeline: WaveformPipeline | None = featurization_pipeline
|
|
88
|
+
self.fixed_property_keys: tuple[str] | list[str] = fixed_property_keys
|
|
89
|
+
|
|
90
|
+
# subclasses can append to this if they want to store more fixed
|
|
91
|
+
# arrays in the output h5 file
|
|
92
|
+
self.fixed_output_data: list[tuple[str, Any]] = [
|
|
93
|
+
("sampling_frequency", self.recording.get_sampling_frequency()),
|
|
94
|
+
("geom", self.recording.get_channel_locations()),
|
|
95
|
+
]
|
|
96
|
+
if channel_index is not None:
|
|
97
|
+
self.fixed_output_data.append(
|
|
98
|
+
("channel_index", self.b.channel_index.numpy(force=True).copy()),
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
self._rgs: local = local()
|
|
102
|
+
|
|
103
|
+
# -- main functions for users to call
|
|
104
|
+
# in practice users will interact with the functions `subtract(...)` in
|
|
105
|
+
# subtract.py and similar functions in the other .py files here, but these
|
|
106
|
+
# are the main API methods for this class
|
|
107
|
+
|
|
108
|
+
def load_or_fit_and_save_models(
|
|
109
|
+
self, save_folder, overwrite=False, computation_cfg=None
|
|
110
|
+
):
|
|
111
|
+
"""Load fitted models from save_folder if possible, or fit and save
|
|
112
|
+
|
|
113
|
+
If the peeler has models that need to be trained, this function ensures
|
|
114
|
+
that the models are fitted and that their fitted parameters are saved
|
|
115
|
+
to `save_folder`
|
|
116
|
+
"""
|
|
117
|
+
save_folder = Path(save_folder)
|
|
118
|
+
if overwrite and save_folder.exists():
|
|
119
|
+
for pt_file in save_folder.glob("*pipeline.pt"):
|
|
120
|
+
pt_file.unlink()
|
|
121
|
+
if self.needs_precompute() or self.needs_fit():
|
|
122
|
+
self.load_models(save_folder)
|
|
123
|
+
if self.needs_precompute() or self.needs_fit():
|
|
124
|
+
if self.needs_precompute():
|
|
125
|
+
self.precompute_models(
|
|
126
|
+
save_folder, overwrite=overwrite, computation_cfg=computation_cfg
|
|
127
|
+
)
|
|
128
|
+
if self.needs_fit():
|
|
129
|
+
save_folder.mkdir(exist_ok=True)
|
|
130
|
+
self.fit_models(
|
|
131
|
+
save_folder, overwrite=overwrite, computation_cfg=computation_cfg
|
|
132
|
+
)
|
|
133
|
+
self.save_models(save_folder)
|
|
134
|
+
assert not self.needs_precompute()
|
|
135
|
+
assert not self.needs_fit()
|
|
136
|
+
|
|
137
|
+
def peel(
|
|
138
|
+
self,
|
|
139
|
+
output_hdf5_filename,
|
|
140
|
+
chunk_starts_samples=None,
|
|
141
|
+
chunk_length_samples=None,
|
|
142
|
+
total_residual_snips=None,
|
|
143
|
+
residual_snips_per_chunk=None,
|
|
144
|
+
stop_after_n_waveforms=None,
|
|
145
|
+
shuffle=False,
|
|
146
|
+
overwrite=False,
|
|
147
|
+
residual_filename=None,
|
|
148
|
+
show_progress=True,
|
|
149
|
+
skip_features=False,
|
|
150
|
+
residual_to_h5=False,
|
|
151
|
+
task_name=None,
|
|
152
|
+
ignore_resuming=False,
|
|
153
|
+
computation_cfg=None,
|
|
154
|
+
):
|
|
155
|
+
"""Run the full (already fitted) peeling and featurization pipeline
|
|
156
|
+
|
|
157
|
+
This gathers all results into an hdf5 file, resuming from where
|
|
158
|
+
it left off in that file if overwrite=False.
|
|
159
|
+
"""
|
|
160
|
+
if self.needs_fit():
|
|
161
|
+
raise ValueError("Peeler needs to be fitted before peeling.")
|
|
162
|
+
|
|
163
|
+
if output_hdf5_filename is None:
|
|
164
|
+
# all of these features would require the h5 file.
|
|
165
|
+
assert ignore_resuming
|
|
166
|
+
assert not residual_to_h5
|
|
167
|
+
assert residual_snips_per_chunk is None
|
|
168
|
+
assert total_residual_snips is None
|
|
169
|
+
|
|
170
|
+
if task_name is None:
|
|
171
|
+
task_name = self.peel_kind
|
|
172
|
+
|
|
173
|
+
# this is -1 if we haven't started yet
|
|
174
|
+
if ignore_resuming:
|
|
175
|
+
last_chunk_start = -1
|
|
176
|
+
resids_so_far = 0
|
|
177
|
+
elif output_hdf5_filename is not None:
|
|
178
|
+
last_chunk_start, resids_so_far = self.check_resuming(
|
|
179
|
+
output_hdf5_filename, overwrite=overwrite
|
|
180
|
+
)
|
|
181
|
+
logger.dartsortdebug(
|
|
182
|
+
f"[{self.__class__.__name__}:{task_name}] Resuming at {last_chunk_start=}."
|
|
183
|
+
)
|
|
184
|
+
else:
|
|
185
|
+
assert False
|
|
186
|
+
|
|
187
|
+
# figure out which chunks to process, and exit early if already done
|
|
188
|
+
chunk_starts_samples = self.get_chunk_starts(
|
|
189
|
+
chunk_starts_samples=chunk_starts_samples,
|
|
190
|
+
)
|
|
191
|
+
chunk_starts_samples = np.asarray(chunk_starts_samples, dtype=int)
|
|
192
|
+
if shuffle:
|
|
193
|
+
assert overwrite or last_chunk_start < 0
|
|
194
|
+
self.fit_subsampling_random_state.shuffle(chunk_starts_samples)
|
|
195
|
+
n_chunks_orig = len(chunk_starts_samples)
|
|
196
|
+
chunks_to_do = [
|
|
197
|
+
start for start in chunk_starts_samples if start > last_chunk_start
|
|
198
|
+
]
|
|
199
|
+
|
|
200
|
+
if total_residual_snips is not None:
|
|
201
|
+
assert residual_snips_per_chunk is None
|
|
202
|
+
resids_remaining = total_residual_snips - resids_so_far
|
|
203
|
+
residual_snips_per_chunk = divide_randomly(
|
|
204
|
+
resids_remaining, len(chunks_to_do), self.fit_subsampling_random_state
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
if residual_snips_per_chunk is None:
|
|
208
|
+
residual_snips_per_chunk = repeat(None)
|
|
209
|
+
elif isinstance(residual_snips_per_chunk, int):
|
|
210
|
+
residual_to_h5 = bool(residual_snips_per_chunk)
|
|
211
|
+
residual_snips_per_chunk = repeat(residual_snips_per_chunk)
|
|
212
|
+
else:
|
|
213
|
+
residual_to_h5 = True
|
|
214
|
+
assert len(residual_snips_per_chunk) == len(chunks_to_do)
|
|
215
|
+
|
|
216
|
+
jobs = list(zip(chunks_to_do, residual_snips_per_chunk))
|
|
217
|
+
|
|
218
|
+
if not chunks_to_do:
|
|
219
|
+
return output_hdf5_filename
|
|
220
|
+
save_residual = residual_filename is not None
|
|
221
|
+
compute_residual = save_residual or residual_to_h5
|
|
222
|
+
if chunk_length_samples is None:
|
|
223
|
+
chunk_length_samples = self.chunk_length_samples
|
|
224
|
+
if computation_cfg is None:
|
|
225
|
+
computation_cfg = job_util.get_global_computation_config()
|
|
226
|
+
|
|
227
|
+
to_cpu = output_hdf5_filename is not None
|
|
228
|
+
|
|
229
|
+
# main peeling loop
|
|
230
|
+
# wrap in try/finally to ensure file handles get closed if there
|
|
231
|
+
# is some unforseen error
|
|
232
|
+
try:
|
|
233
|
+
n_jobs, Executor, context, rank_queue, is_local = pool_from_cfg( # type: ignore
|
|
234
|
+
computation_cfg, with_rank_queue=True, check_local=True
|
|
235
|
+
)
|
|
236
|
+
with Executor(
|
|
237
|
+
max_workers=n_jobs,
|
|
238
|
+
mp_context=context,
|
|
239
|
+
initializer=_peeler_process_init,
|
|
240
|
+
initargs=(
|
|
241
|
+
self,
|
|
242
|
+
computation_cfg.actual_device(),
|
|
243
|
+
rank_queue,
|
|
244
|
+
compute_residual,
|
|
245
|
+
skip_features,
|
|
246
|
+
chunk_length_samples,
|
|
247
|
+
is_local,
|
|
248
|
+
to_cpu,
|
|
249
|
+
),
|
|
250
|
+
) as pool:
|
|
251
|
+
if is_local:
|
|
252
|
+
self.to(computation_cfg.actual_device())
|
|
253
|
+
|
|
254
|
+
# launch the jobs and wrap in a progress bar
|
|
255
|
+
results = pool.map(_peeler_process_job, jobs)
|
|
256
|
+
if show_progress:
|
|
257
|
+
n_sec_chunk = (
|
|
258
|
+
chunk_length_samples / self.recording.get_sampling_frequency()
|
|
259
|
+
)
|
|
260
|
+
dtag = computation_cfg.actual_device().type
|
|
261
|
+
results = tqdm(
|
|
262
|
+
results,
|
|
263
|
+
total=n_chunks_orig,
|
|
264
|
+
initial=n_chunks_orig - len(chunks_to_do),
|
|
265
|
+
smoothing=0,
|
|
266
|
+
desc=f"{task_name}:{dtag} {n_sec_chunk:.1f}s/it [spk/it=%%%]",
|
|
267
|
+
mininterval=0.25,
|
|
268
|
+
)
|
|
269
|
+
else:
|
|
270
|
+
n_sec_chunk = None
|
|
271
|
+
|
|
272
|
+
# construct h5 after forking to avoid pickling it
|
|
273
|
+
with self.initialize_files(
|
|
274
|
+
output_hdf5_filename,
|
|
275
|
+
residual_filename=residual_filename,
|
|
276
|
+
overwrite=overwrite,
|
|
277
|
+
skip_features=skip_features,
|
|
278
|
+
residual_to_h5=residual_to_h5,
|
|
279
|
+
) as (
|
|
280
|
+
output_h5,
|
|
281
|
+
h5_spike_datasets,
|
|
282
|
+
residual_file,
|
|
283
|
+
n_spikes,
|
|
284
|
+
):
|
|
285
|
+
batch_count = 0
|
|
286
|
+
try:
|
|
287
|
+
for result, chunk_start_samples in zip(results, chunks_to_do):
|
|
288
|
+
n_new_spikes = self.gather_chunk_result(
|
|
289
|
+
n_spikes,
|
|
290
|
+
chunk_start_samples,
|
|
291
|
+
result,
|
|
292
|
+
h5_spike_datasets,
|
|
293
|
+
output_h5,
|
|
294
|
+
residual_file,
|
|
295
|
+
residual_to_h5,
|
|
296
|
+
ignore_resuming,
|
|
297
|
+
skip_features,
|
|
298
|
+
)
|
|
299
|
+
batch_count += 1
|
|
300
|
+
n_spikes += n_new_spikes
|
|
301
|
+
if show_progress:
|
|
302
|
+
desc = f"{task_name}"
|
|
303
|
+
if not skip_features:
|
|
304
|
+
assert n_sec_chunk is not None
|
|
305
|
+
desc += f" [spk/{n_sec_chunk:g}s={n_spikes / batch_count:0.1f}]"
|
|
306
|
+
results.set_description(desc, refresh=False) # type: ignore
|
|
307
|
+
if not skip_features and (
|
|
308
|
+
stop_after_n_waveforms
|
|
309
|
+
and n_spikes >= stop_after_n_waveforms
|
|
310
|
+
):
|
|
311
|
+
pool.shutdown(cancel_futures=True)
|
|
312
|
+
except CancelledError:
|
|
313
|
+
if show_progress:
|
|
314
|
+
results.write( # type: ignore
|
|
315
|
+
f"Got {n_spikes} spikes, enough to stop early."
|
|
316
|
+
)
|
|
317
|
+
finally:
|
|
318
|
+
self.to("cpu")
|
|
319
|
+
|
|
320
|
+
# we very much do not want to leak self into a global and have its
|
|
321
|
+
# memory tied up while dartsort wants to go do other stuff
|
|
322
|
+
global _peeler_process_context
|
|
323
|
+
del _peeler_process_context.ctx
|
|
324
|
+
_peeler_process_context.ctx = None
|
|
325
|
+
|
|
326
|
+
return output_hdf5_filename
|
|
327
|
+
|
|
328
|
+
# -- methods for subclasses to override
|
|
329
|
+
|
|
330
|
+
def peel_chunk(
|
|
331
|
+
self,
|
|
332
|
+
traces,
|
|
333
|
+
*,
|
|
334
|
+
chunk_start_samples=0,
|
|
335
|
+
left_margin=0,
|
|
336
|
+
right_margin=0,
|
|
337
|
+
return_residual=False,
|
|
338
|
+
return_waveforms=True,
|
|
339
|
+
) -> "PeelingBatchResult":
|
|
340
|
+
# subclasses should implement this method
|
|
341
|
+
|
|
342
|
+
# they should return a dictionary with keys:
|
|
343
|
+
# - n_spikes
|
|
344
|
+
# - collisioncleaned_waveforms
|
|
345
|
+
# - times_samples (relative to start of traces)
|
|
346
|
+
# - channels
|
|
347
|
+
# - residual if requested
|
|
348
|
+
# - arbitrary subclass-specific stuff which should have keys in out_datasets()
|
|
349
|
+
|
|
350
|
+
# data for spikes in the margin should not be returned
|
|
351
|
+
# these spikes will have been processed by the neighboring chunk
|
|
352
|
+
|
|
353
|
+
raise NotImplementedError
|
|
354
|
+
|
|
355
|
+
def peeling_needs_fit(self) -> bool:
|
|
356
|
+
return False
|
|
357
|
+
|
|
358
|
+
def peeling_needs_precompute(self) -> bool:
|
|
359
|
+
return False
|
|
360
|
+
|
|
361
|
+
def precompute_peeling_data(
|
|
362
|
+
self, save_folder, overwrite=False, computation_cfg=None
|
|
363
|
+
):
|
|
364
|
+
# subclasses should override if they need to cache data for peeling
|
|
365
|
+
# runs before fit_peeler_models()
|
|
366
|
+
pass
|
|
367
|
+
|
|
368
|
+
def fit_peeler_models(self, save_folder, tmp_dir=None, computation_cfg=None):
|
|
369
|
+
# subclasses should override if they need to fit models for peeling
|
|
370
|
+
assert not self.peeling_needs_fit()
|
|
371
|
+
|
|
372
|
+
def precompute_peeler_models(
|
|
373
|
+
self, save_folder, overwrite=False, computation_cfg=None
|
|
374
|
+
):
|
|
375
|
+
if self.peeling_needs_precompute():
|
|
376
|
+
self.precompute_peeling_data(
|
|
377
|
+
save_folder=save_folder,
|
|
378
|
+
overwrite=overwrite,
|
|
379
|
+
computation_cfg=computation_cfg,
|
|
380
|
+
)
|
|
381
|
+
assert not self.peeling_needs_precompute()
|
|
382
|
+
|
|
383
|
+
# subclasses can add to this list
|
|
384
|
+
# for each dataset in this list, an output dataset in the
|
|
385
|
+
# hdf5 (of .peel()) will be created with this dtype and with
|
|
386
|
+
# shape (N_spikes, *shape_per_spike)
|
|
387
|
+
# datasets will also be created corresponding to features
|
|
388
|
+
# in featurization_pipeline
|
|
389
|
+
def out_datasets(self):
|
|
390
|
+
datasets = [
|
|
391
|
+
SpikeDataset(name="times_samples", shape_per_spike=(), dtype=np.int64),
|
|
392
|
+
SpikeDataset(name="times_seconds", shape_per_spike=(), dtype=np.float64),
|
|
393
|
+
SpikeDataset(name="channels", shape_per_spike=(), dtype=np.int64),
|
|
394
|
+
]
|
|
395
|
+
if self.featurization_pipeline is not None:
|
|
396
|
+
datasets.extend(self.featurization_pipeline.spike_datasets())
|
|
397
|
+
return datasets
|
|
398
|
+
|
|
399
|
+
# -- utility methods which users likely won't touch
|
|
400
|
+
|
|
401
|
+
def featurize_collisioncleaned_waveforms(
|
|
402
|
+
self, collisioncleaned_waveforms, **fixed_properties
|
|
403
|
+
):
|
|
404
|
+
if not self.featurization_pipeline:
|
|
405
|
+
return {}
|
|
406
|
+
|
|
407
|
+
waveforms, features = self.featurization_pipeline(
|
|
408
|
+
collisioncleaned_waveforms, **fixed_properties
|
|
409
|
+
)
|
|
410
|
+
return features
|
|
411
|
+
|
|
412
|
+
def process_chunk(
|
|
413
|
+
self,
|
|
414
|
+
chunk_start_samples: int,
|
|
415
|
+
*,
|
|
416
|
+
chunk_end_samples: int | None = None,
|
|
417
|
+
return_residual: bool = False,
|
|
418
|
+
skip_features: bool = False,
|
|
419
|
+
n_resid_snips: int | None = None,
|
|
420
|
+
to_cpu: bool = True,
|
|
421
|
+
) -> "PeelingBatchResult":
|
|
422
|
+
"""Grab, peel, and featurize a chunk, returning a dict of numpy arrays
|
|
423
|
+
|
|
424
|
+
Main function called in peeling workers
|
|
425
|
+
"""
|
|
426
|
+
if chunk_end_samples is None:
|
|
427
|
+
chunk_end_samples = chunk_start_samples + self.chunk_length_samples
|
|
428
|
+
chunk_end_samples = min(self.recording.get_num_samples(), chunk_end_samples)
|
|
429
|
+
chunk, left_margin, right_margin = get_chunk_with_margin(
|
|
430
|
+
self.recording._recording_segments[0],
|
|
431
|
+
start_frame=chunk_start_samples,
|
|
432
|
+
end_frame=chunk_end_samples,
|
|
433
|
+
channel_indices=None,
|
|
434
|
+
margin=self.chunk_margin_samples,
|
|
435
|
+
)
|
|
436
|
+
chunk = torch.tensor(
|
|
437
|
+
chunk, device=self.b.channel_index.device, dtype=self.dtype
|
|
438
|
+
)
|
|
439
|
+
return_waveforms = not skip_features and bool(self.featurization_pipeline)
|
|
440
|
+
peel_result = self.peel_chunk(
|
|
441
|
+
chunk,
|
|
442
|
+
chunk_start_samples=chunk_start_samples,
|
|
443
|
+
left_margin=left_margin,
|
|
444
|
+
right_margin=right_margin,
|
|
445
|
+
return_waveforms=return_waveforms,
|
|
446
|
+
return_residual=return_residual or bool(n_resid_snips),
|
|
447
|
+
)
|
|
448
|
+
|
|
449
|
+
if peel_result["n_spikes"] > 0 and return_waveforms:
|
|
450
|
+
fixed_properties = {k: peel_result[k] for k in self.fixed_property_keys}
|
|
451
|
+
features = self.featurize_collisioncleaned_waveforms(
|
|
452
|
+
peel_result["collisioncleaned_waveforms"], **fixed_properties
|
|
453
|
+
)
|
|
454
|
+
else:
|
|
455
|
+
features = {}
|
|
456
|
+
|
|
457
|
+
# a user who wants these must featurize with a waveform node
|
|
458
|
+
# then they'll end up in `features`
|
|
459
|
+
if "collisioncleaned_waveforms" in peel_result:
|
|
460
|
+
del peel_result["collisioncleaned_waveforms"]
|
|
461
|
+
|
|
462
|
+
assert not any(k in features for k in peel_result)
|
|
463
|
+
chunk_result = {**peel_result, **features}
|
|
464
|
+
if to_cpu:
|
|
465
|
+
chunk_result = {
|
|
466
|
+
k: v.cpu() if torch.is_tensor(v) else v for k, v in chunk_result.items()
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
if "residual" in peel_result:
|
|
470
|
+
if torch.is_tensor(peel_result["residual"]):
|
|
471
|
+
peel_result["residual"] = peel_result["residual"].cpu() # type: ignore
|
|
472
|
+
|
|
473
|
+
# add times in seconds
|
|
474
|
+
segment = self.recording._recording_segments[0]
|
|
475
|
+
chunk_result["chunk_start_seconds"] = segment.sample_index_to_time(
|
|
476
|
+
chunk_start_samples
|
|
477
|
+
)
|
|
478
|
+
if peel_result["n_spikes"] and to_cpu:
|
|
479
|
+
chunk_result["times_seconds"] = segment.sample_index_to_time(
|
|
480
|
+
chunk_result["times_samples"]
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
if n_resid_snips:
|
|
484
|
+
chunk_result["resid_snips"], resid_times_samples = extract_random_snips(
|
|
485
|
+
self.rg,
|
|
486
|
+
peel_result["residual"],
|
|
487
|
+
n_resid_snips,
|
|
488
|
+
self.spike_length_samples,
|
|
489
|
+
)
|
|
490
|
+
chunk_result["residual_times_seconds"] = (
|
|
491
|
+
self.recording.sample_index_to_time(
|
|
492
|
+
chunk_start_samples + resid_times_samples
|
|
493
|
+
)
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
return chunk_result # type: ignore
|
|
497
|
+
|
|
498
|
+
def gather_chunk_result(
|
|
499
|
+
self,
|
|
500
|
+
cur_n_spikes,
|
|
501
|
+
chunk_start_samples,
|
|
502
|
+
chunk_result,
|
|
503
|
+
h5_spike_datasets,
|
|
504
|
+
output_h5,
|
|
505
|
+
residual_file,
|
|
506
|
+
residual_to_h5,
|
|
507
|
+
ignore_resuming,
|
|
508
|
+
skip_features,
|
|
509
|
+
):
|
|
510
|
+
# delay keyboard interrupts so we don't write half a batch
|
|
511
|
+
# of data and leave files in an invalid state after ^C
|
|
512
|
+
# not that something else couldn't happen...
|
|
513
|
+
with delay_keyboard_interrupt:
|
|
514
|
+
if not ignore_resuming:
|
|
515
|
+
output_h5["last_chunk_start"][()] = chunk_start_samples
|
|
516
|
+
|
|
517
|
+
if residual_file is not None:
|
|
518
|
+
chunk_result["residual"].tofile(residual_file)
|
|
519
|
+
|
|
520
|
+
if "resid_snips" in chunk_result:
|
|
521
|
+
n_residuals = len(output_h5["residual"])
|
|
522
|
+
n_new_res = len(chunk_result["resid_snips"])
|
|
523
|
+
assert chunk_result["residual_times_seconds"].shape == (n_new_res,)
|
|
524
|
+
output_h5["residual"].resize(n_residuals + n_new_res, axis=0)
|
|
525
|
+
output_h5["residual"][n_residuals:] = chunk_result["resid_snips"]
|
|
526
|
+
output_h5["residual_times_seconds"].resize(
|
|
527
|
+
n_residuals + n_new_res, axis=0
|
|
528
|
+
)
|
|
529
|
+
output_h5["residual_times_seconds"][n_residuals:] = chunk_result[
|
|
530
|
+
"residual_times_seconds"
|
|
531
|
+
]
|
|
532
|
+
|
|
533
|
+
if skip_features:
|
|
534
|
+
return 0
|
|
535
|
+
|
|
536
|
+
n_new_spikes = chunk_result["n_spikes"]
|
|
537
|
+
if not n_new_spikes:
|
|
538
|
+
return 0
|
|
539
|
+
|
|
540
|
+
for ds in self.out_datasets():
|
|
541
|
+
h5ds = h5_spike_datasets[ds.name]
|
|
542
|
+
h5ds.resize(cur_n_spikes + n_new_spikes, axis=0)
|
|
543
|
+
h5ds[cur_n_spikes:] = chunk_result[ds.name]
|
|
544
|
+
|
|
545
|
+
return n_new_spikes
|
|
546
|
+
|
|
547
|
+
def needs_fit(self):
|
|
548
|
+
if self.peeling_needs_fit():
|
|
549
|
+
return True
|
|
550
|
+
if self.featurization_pipeline is not None:
|
|
551
|
+
return self.featurization_pipeline.needs_fit()
|
|
552
|
+
return False
|
|
553
|
+
|
|
554
|
+
def needs_precompute(self):
|
|
555
|
+
if self.peeling_needs_precompute():
|
|
556
|
+
return True
|
|
557
|
+
if self.featurization_pipeline is not None:
|
|
558
|
+
return self.featurization_pipeline.needs_precompute()
|
|
559
|
+
return False
|
|
560
|
+
|
|
561
|
+
def fit_models(
|
|
562
|
+
self, save_folder, tmp_dir=None, overwrite=False, computation_cfg=None
|
|
563
|
+
):
|
|
564
|
+
with torch.no_grad():
|
|
565
|
+
if self.peeling_needs_fit():
|
|
566
|
+
self.precompute_peeling_data(
|
|
567
|
+
save_folder=save_folder,
|
|
568
|
+
overwrite=overwrite,
|
|
569
|
+
computation_cfg=computation_cfg,
|
|
570
|
+
)
|
|
571
|
+
self.fit_peeler_models(
|
|
572
|
+
save_folder=save_folder,
|
|
573
|
+
tmp_dir=tmp_dir,
|
|
574
|
+
computation_cfg=computation_cfg,
|
|
575
|
+
)
|
|
576
|
+
self.fit_featurization_pipeline(
|
|
577
|
+
tmp_dir=tmp_dir,
|
|
578
|
+
computation_cfg=computation_cfg,
|
|
579
|
+
)
|
|
580
|
+
assert not self.needs_fit()
|
|
581
|
+
|
|
582
|
+
def precompute_models(self, save_folder, overwrite=False, computation_cfg=None):
|
|
583
|
+
if self.peeling_needs_precompute():
|
|
584
|
+
self.precompute_peeler_models(
|
|
585
|
+
save_folder=save_folder,
|
|
586
|
+
overwrite=overwrite,
|
|
587
|
+
computation_cfg=computation_cfg,
|
|
588
|
+
)
|
|
589
|
+
if self.featurization_pipeline is None:
|
|
590
|
+
return
|
|
591
|
+
self.featurization_pipeline.precompute()
|
|
592
|
+
|
|
593
|
+
def fit_featurization_pipeline(self, tmp_dir=None, computation_cfg=None):
|
|
594
|
+
if self.featurization_pipeline is None:
|
|
595
|
+
return
|
|
596
|
+
|
|
597
|
+
if not self.featurization_pipeline.needs_fit():
|
|
598
|
+
return
|
|
599
|
+
|
|
600
|
+
if computation_cfg is None:
|
|
601
|
+
computation_cfg = job_util.get_global_computation_config()
|
|
602
|
+
device = computation_cfg.actual_device()
|
|
603
|
+
|
|
604
|
+
# disable self's featurization pipeline, replacing it with a waveform
|
|
605
|
+
# saving node. then, we'll use those waveforms to fit the original
|
|
606
|
+
featurization_pipeline = self.featurization_pipeline
|
|
607
|
+
self.featurization_pipeline = WaveformPipeline.from_class_names_and_kwargs(
|
|
608
|
+
self.recording.get_channel_locations(),
|
|
609
|
+
self.channel_index,
|
|
610
|
+
[
|
|
611
|
+
("Voltage", {"name": "peeled_voltages_fit"}),
|
|
612
|
+
(
|
|
613
|
+
"Waveform",
|
|
614
|
+
{
|
|
615
|
+
"name": "peeled_waveforms_fit",
|
|
616
|
+
"spike_length_samples": self.spike_length_samples,
|
|
617
|
+
},
|
|
618
|
+
),
|
|
619
|
+
],
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
with tempfile.TemporaryDirectory(dir=tmp_dir) as temp_dir:
|
|
623
|
+
temp_hdf5_filename = Path(temp_dir) / "peeler_fit.h5"
|
|
624
|
+
waveforms = fixed_properties = None
|
|
625
|
+
try:
|
|
626
|
+
self.run_subsampled_peeling(
|
|
627
|
+
temp_hdf5_filename,
|
|
628
|
+
computation_cfg=computation_cfg,
|
|
629
|
+
task_name="Load examples for feature fitting",
|
|
630
|
+
)
|
|
631
|
+
|
|
632
|
+
# fit featurization pipeline and reassign
|
|
633
|
+
# work in a try finally so we can delete the temp file
|
|
634
|
+
# in case of an issue or a keyboard interrupt
|
|
635
|
+
waveforms, fixed_properties = subsample_waveforms(
|
|
636
|
+
temp_hdf5_filename,
|
|
637
|
+
fit_sampling=self.fit_sampling,
|
|
638
|
+
random_state=self.fit_subsampling_random_state,
|
|
639
|
+
n_waveforms_fit=self.n_waveforms_fit,
|
|
640
|
+
fit_max_reweighting=self.fit_max_reweighting,
|
|
641
|
+
voltages_dataset_name="peeled_voltages_fit",
|
|
642
|
+
waveforms_dataset_name="peeled_waveforms_fit",
|
|
643
|
+
fixed_property_keys=self.fixed_property_keys,
|
|
644
|
+
device=device,
|
|
645
|
+
)
|
|
646
|
+
if not len(waveforms):
|
|
647
|
+
raise ValueError(f"Found no spikes when trying to fit featurizers.")
|
|
648
|
+
|
|
649
|
+
featurization_pipeline = featurization_pipeline.to(device)
|
|
650
|
+
featurization_pipeline.fit(
|
|
651
|
+
recording=self.recording,
|
|
652
|
+
waveforms=waveforms,
|
|
653
|
+
**fixed_properties,
|
|
654
|
+
)
|
|
655
|
+
assert getrefcount(waveforms) == 2
|
|
656
|
+
featurization_pipeline = featurization_pipeline.to("cpu")
|
|
657
|
+
self.featurization_pipeline = featurization_pipeline
|
|
658
|
+
finally:
|
|
659
|
+
self.to("cpu")
|
|
660
|
+
if temp_hdf5_filename.exists():
|
|
661
|
+
temp_hdf5_filename.unlink()
|
|
662
|
+
del waveforms, fixed_properties
|
|
663
|
+
gc.collect()
|
|
664
|
+
|
|
665
|
+
def get_chunk_starts(
|
|
666
|
+
self,
|
|
667
|
+
chunk_starts_samples=None,
|
|
668
|
+
chunk_length_samples=None,
|
|
669
|
+
t_start=None,
|
|
670
|
+
t_end=None,
|
|
671
|
+
subsampled=False,
|
|
672
|
+
n_chunks=None,
|
|
673
|
+
ordered=False,
|
|
674
|
+
skip_last=False,
|
|
675
|
+
):
|
|
676
|
+
if chunk_starts_samples is not None:
|
|
677
|
+
return chunk_starts_samples
|
|
678
|
+
if chunk_length_samples is None:
|
|
679
|
+
chunk_length_samples = self.chunk_length_samples
|
|
680
|
+
|
|
681
|
+
T_samples = self.recording.get_num_samples()
|
|
682
|
+
if t_end is None:
|
|
683
|
+
t_end = T_samples
|
|
684
|
+
if t_start is None:
|
|
685
|
+
t_start = 0
|
|
686
|
+
chunk_starts_samples = range(t_start, t_end, chunk_length_samples)
|
|
687
|
+
if skip_last:
|
|
688
|
+
chunk_starts_samples = list(chunk_starts_samples)
|
|
689
|
+
if t_end - chunk_starts_samples[-1] < chunk_length_samples:
|
|
690
|
+
chunk_starts_samples = chunk_starts_samples[:-1]
|
|
691
|
+
|
|
692
|
+
if not subsampled:
|
|
693
|
+
return chunk_starts_samples
|
|
694
|
+
|
|
695
|
+
if n_chunks is None:
|
|
696
|
+
chunks_per_second = self.recording.sampling_frequency / chunk_length_samples
|
|
697
|
+
n_chunks = int(np.ceil(self.n_seconds_fit * chunks_per_second))
|
|
698
|
+
|
|
699
|
+
# make a random subset of chunks to use for fitting
|
|
700
|
+
rg = np.random.default_rng(self.fit_subsampling_random_state)
|
|
701
|
+
self.fit_subsampling_random_state = rg
|
|
702
|
+
chunk_starts_samples = rg.choice(
|
|
703
|
+
chunk_starts_samples,
|
|
704
|
+
size=min(len(chunk_starts_samples), n_chunks),
|
|
705
|
+
replace=False,
|
|
706
|
+
)
|
|
707
|
+
if ordered:
|
|
708
|
+
chunk_starts_samples.sort()
|
|
709
|
+
return chunk_starts_samples
|
|
710
|
+
|
|
711
|
+
def run_subsampled_peeling(
|
|
712
|
+
self,
|
|
713
|
+
hdf5_filename,
|
|
714
|
+
chunk_length_samples=None,
|
|
715
|
+
residual_to_h5=False,
|
|
716
|
+
skip_features=False,
|
|
717
|
+
ignore_resuming=False,
|
|
718
|
+
n_chunks=None,
|
|
719
|
+
t_start=None,
|
|
720
|
+
t_end=None,
|
|
721
|
+
computation_cfg=None,
|
|
722
|
+
task_name=None,
|
|
723
|
+
overwrite=True,
|
|
724
|
+
ordered=False,
|
|
725
|
+
skip_last=False,
|
|
726
|
+
show_progress=True,
|
|
727
|
+
):
|
|
728
|
+
# run peeling on these chunks to the temp folder
|
|
729
|
+
chunk_starts = self.get_chunk_starts(
|
|
730
|
+
subsampled=True,
|
|
731
|
+
chunk_length_samples=chunk_length_samples,
|
|
732
|
+
t_start=t_start,
|
|
733
|
+
t_end=t_end,
|
|
734
|
+
n_chunks=n_chunks,
|
|
735
|
+
ordered=ordered,
|
|
736
|
+
skip_last=skip_last,
|
|
737
|
+
)
|
|
738
|
+
return self.peel(
|
|
739
|
+
hdf5_filename,
|
|
740
|
+
chunk_starts_samples=chunk_starts,
|
|
741
|
+
chunk_length_samples=chunk_length_samples,
|
|
742
|
+
stop_after_n_waveforms=self.max_waveforms_fit,
|
|
743
|
+
ignore_resuming=ignore_resuming,
|
|
744
|
+
residual_to_h5=residual_to_h5,
|
|
745
|
+
skip_features=skip_features,
|
|
746
|
+
computation_cfg=computation_cfg,
|
|
747
|
+
overwrite=overwrite,
|
|
748
|
+
task_name=task_name,
|
|
749
|
+
show_progress=show_progress,
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
def save_models(self, save_folder):
|
|
753
|
+
if self.featurization_pipeline is not None:
|
|
754
|
+
Path(save_folder).mkdir(exist_ok=True)
|
|
755
|
+
torch.save(
|
|
756
|
+
self.featurization_pipeline.state_dict(),
|
|
757
|
+
Path(save_folder) / "featurization_pipeline.pt",
|
|
758
|
+
)
|
|
759
|
+
|
|
760
|
+
def load_models(self, save_folder):
|
|
761
|
+
if not save_folder.exists():
|
|
762
|
+
return
|
|
763
|
+
|
|
764
|
+
feats_pt = Path(save_folder) / "featurization_pipeline.pt"
|
|
765
|
+
if feats_pt.exists():
|
|
766
|
+
assert self.featurization_pipeline is not None
|
|
767
|
+
state_dict = torch.load(feats_pt)
|
|
768
|
+
self.featurization_pipeline.load_state_dict(state_dict)
|
|
769
|
+
|
|
770
|
+
def check_resuming(self, output_hdf5_filename, overwrite=False) -> tuple[int, int]:
|
|
771
|
+
output_hdf5_filename = Path(output_hdf5_filename)
|
|
772
|
+
exists = output_hdf5_filename.exists()
|
|
773
|
+
last_chunk_start = -1
|
|
774
|
+
residual_snips_so_far = 0
|
|
775
|
+
if exists and not overwrite:
|
|
776
|
+
with h5py.File(output_hdf5_filename, "r", locking=False) as h5:
|
|
777
|
+
last_chunk_start: int = h5["last_chunk_start"][()] # type: ignore
|
|
778
|
+
residual_snips_so_far = 0
|
|
779
|
+
if "residual" in h5:
|
|
780
|
+
residual_snips_so_far = len(h5["residual"]) # type: ignore
|
|
781
|
+
return last_chunk_start, residual_snips_so_far
|
|
782
|
+
|
|
783
|
+
@contextmanager
|
|
784
|
+
def initialize_files(
|
|
785
|
+
self,
|
|
786
|
+
output_hdf5_filename,
|
|
787
|
+
residual_filename=None,
|
|
788
|
+
overwrite=False,
|
|
789
|
+
chunk_size=1024,
|
|
790
|
+
# could also do:
|
|
791
|
+
# libver=("earliest", "v110"),
|
|
792
|
+
libver="latest",
|
|
793
|
+
residual_to_h5=False,
|
|
794
|
+
skip_features=False,
|
|
795
|
+
):
|
|
796
|
+
"""Create, overwrite, or re-open output files"""
|
|
797
|
+
if output_hdf5_filename is None:
|
|
798
|
+
assert residual_filename is None
|
|
799
|
+
assert not residual_to_h5
|
|
800
|
+
# this is not usually the case, but it is used by the
|
|
801
|
+
# RunningTemplates peeler.
|
|
802
|
+
yield None, None, None, 0
|
|
803
|
+
return
|
|
804
|
+
|
|
805
|
+
output_hdf5_filename = Path(output_hdf5_filename)
|
|
806
|
+
exists = output_hdf5_filename.exists()
|
|
807
|
+
n_spikes = 0
|
|
808
|
+
if exists and overwrite:
|
|
809
|
+
output_hdf5_filename.unlink()
|
|
810
|
+
output_h5 = h5py.File(output_hdf5_filename, "w", libver=libver)
|
|
811
|
+
output_h5.create_dataset("last_chunk_start", data=-1, dtype=np.int64)
|
|
812
|
+
elif exists:
|
|
813
|
+
# exists and not overwrite
|
|
814
|
+
output_h5 = h5py.File(output_hdf5_filename, "r+", libver=libver)
|
|
815
|
+
n_spikes = len(output_h5["times_samples"]) # type: ignore
|
|
816
|
+
else:
|
|
817
|
+
# didn't exist, so overwrite does not matter
|
|
818
|
+
output_h5 = h5py.File(output_hdf5_filename, "w", libver=libver)
|
|
819
|
+
output_h5.create_dataset("last_chunk_start", data=-1, dtype=np.int64)
|
|
820
|
+
last_chunk_start: int = output_h5["last_chunk_start"][()] # type: ignore
|
|
821
|
+
|
|
822
|
+
# write some fixed arrays that are useful to have around
|
|
823
|
+
for name, value in self.fixed_output_data:
|
|
824
|
+
if name not in output_h5:
|
|
825
|
+
output_h5.create_dataset(name, data=value)
|
|
826
|
+
|
|
827
|
+
# create per-spike datasets
|
|
828
|
+
# use chunks to support growing the dataset as we find spikes
|
|
829
|
+
h5_spike_datasets = None
|
|
830
|
+
if not skip_features:
|
|
831
|
+
h5_spike_datasets = {}
|
|
832
|
+
for ds in self.out_datasets():
|
|
833
|
+
if ds.name in output_h5:
|
|
834
|
+
h5_spike_datasets[ds.name] = output_h5[ds.name]
|
|
835
|
+
else:
|
|
836
|
+
h5_spike_datasets[ds.name] = output_h5.create_dataset(
|
|
837
|
+
ds.name,
|
|
838
|
+
dtype=ds.dtype,
|
|
839
|
+
shape=(n_spikes, *ds.shape_per_spike),
|
|
840
|
+
maxshape=(None, *ds.shape_per_spike),
|
|
841
|
+
chunks=(chunk_size, *ds.shape_per_spike),
|
|
842
|
+
)
|
|
843
|
+
|
|
844
|
+
if residual_to_h5:
|
|
845
|
+
if "residual" not in output_h5:
|
|
846
|
+
n_chans = self.recording.get_num_channels()
|
|
847
|
+
output_h5.create_dataset(
|
|
848
|
+
"residual",
|
|
849
|
+
dtype=self.np_dtype,
|
|
850
|
+
shape=(0, self.spike_length_samples, n_chans),
|
|
851
|
+
maxshape=(None, self.spike_length_samples, n_chans),
|
|
852
|
+
chunks=(chunk_size, self.spike_length_samples, n_chans),
|
|
853
|
+
)
|
|
854
|
+
if "residual_times_seconds" not in output_h5:
|
|
855
|
+
output_h5.create_dataset(
|
|
856
|
+
"residual_times_seconds",
|
|
857
|
+
dtype=float,
|
|
858
|
+
shape=(0,),
|
|
859
|
+
maxshape=(None,),
|
|
860
|
+
chunks=(chunk_size,),
|
|
861
|
+
)
|
|
862
|
+
|
|
863
|
+
# residual file ignore/open/overwrite logic
|
|
864
|
+
save_residual = residual_filename is not None
|
|
865
|
+
residual_file = None
|
|
866
|
+
if save_residual:
|
|
867
|
+
residual_mode = "wb"
|
|
868
|
+
if last_chunk_start >= 0:
|
|
869
|
+
residual_mode = "ab"
|
|
870
|
+
assert Path(residual_filename).exists()
|
|
871
|
+
residual_file = open(residual_filename, mode=residual_mode)
|
|
872
|
+
|
|
873
|
+
try:
|
|
874
|
+
yield (
|
|
875
|
+
output_h5,
|
|
876
|
+
h5_spike_datasets,
|
|
877
|
+
residual_file,
|
|
878
|
+
n_spikes,
|
|
879
|
+
)
|
|
880
|
+
finally:
|
|
881
|
+
self.to("cpu")
|
|
882
|
+
output_h5.close()
|
|
883
|
+
if save_residual:
|
|
884
|
+
assert residual_file is not None
|
|
885
|
+
residual_file.close()
|
|
886
|
+
|
|
887
|
+
# -- thread-local rngs. they're not thread safe, and locals can't be pickled
|
|
888
|
+
|
|
889
|
+
def __getstate__(self):
|
|
890
|
+
state = super().__getstate__()
|
|
891
|
+
del state["_rgs"]
|
|
892
|
+
return state
|
|
893
|
+
|
|
894
|
+
def __setstate__(self, state):
|
|
895
|
+
super().__setstate__(state)
|
|
896
|
+
self._rgs = local()
|
|
897
|
+
|
|
898
|
+
@property
|
|
899
|
+
def rg(self):
|
|
900
|
+
if not hasattr(self._rgs, "rg"):
|
|
901
|
+
with _lock:
|
|
902
|
+
self._rgs.rg = self.fit_subsampling_random_state.spawn(1)[0]
|
|
903
|
+
return self._rgs.rg
|
|
904
|
+
|
|
905
|
+
@staticmethod
|
|
906
|
+
def next_margin(length, factor=10):
|
|
907
|
+
return factor * int(np.ceil(length / factor))
|
|
908
|
+
|
|
909
|
+
def nearest_batch_length(self, target=512):
|
|
910
|
+
factors = divisors(self.chunk_length_samples + 2 * self.chunk_margin_samples)
|
|
911
|
+
factors = np.array(factors)
|
|
912
|
+
return factors[np.abs(factors - target).argmin()]
|
|
913
|
+
|
|
914
|
+
|
|
915
|
+
# -- batch result type stub
|
|
916
|
+
|
|
917
|
+
|
|
918
|
+
try:
|
|
919
|
+
# this isn't supported until 3.15, but it helps the type checker
|
|
920
|
+
# so I'm keeping it in here. everything seems to be fine just setting
|
|
921
|
+
# it to dict in practice in the except clause.
|
|
922
|
+
|
|
923
|
+
class PeelingBatchResult(TypedDict, extra_items=torch.Tensor):
|
|
924
|
+
n_spikes: int
|
|
925
|
+
except TypeError:
|
|
926
|
+
PeelingBatchResult = dict # type: ignore
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
peeling_empty_result = PeelingBatchResult(n_spikes=0)
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
# -- helper functions and objects for parallelism
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
class PeelerProcessContext:
|
|
936
|
+
def __init__(
|
|
937
|
+
self, peeler, compute_residual, skip_features, chunk_length_samples, to_cpu
|
|
938
|
+
):
|
|
939
|
+
self.peeler = peeler
|
|
940
|
+
self.compute_residual = compute_residual
|
|
941
|
+
self.skip_features = skip_features
|
|
942
|
+
self.chunk_length_samples = chunk_length_samples
|
|
943
|
+
self.to_cpu = to_cpu
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
# this state will be set on each thread globally
|
|
947
|
+
_peeler_process_context = local()
|
|
948
|
+
_peeler_process_context.ctx = None
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def _peeler_process_init(
|
|
952
|
+
peeler,
|
|
953
|
+
device,
|
|
954
|
+
rank_queue,
|
|
955
|
+
compute_residual,
|
|
956
|
+
skip_features,
|
|
957
|
+
chunk_length_samples,
|
|
958
|
+
is_local,
|
|
959
|
+
to_cpu,
|
|
960
|
+
):
|
|
961
|
+
global _peeler_process_context
|
|
962
|
+
|
|
963
|
+
# figure out what device to work on
|
|
964
|
+
my_rank = rank_queue.get()
|
|
965
|
+
if device is None:
|
|
966
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
967
|
+
device = torch.device(device)
|
|
968
|
+
if device.type == "cuda" and device.index is None:
|
|
969
|
+
if torch.cuda.device_count() > 1:
|
|
970
|
+
index = my_rank % torch.cuda.device_count()
|
|
971
|
+
device = torch.device("cuda", index=index)
|
|
972
|
+
|
|
973
|
+
# move peeler to device and put everything in inference mode
|
|
974
|
+
if not is_local:
|
|
975
|
+
peeler.to(device)
|
|
976
|
+
peeler.eval()
|
|
977
|
+
|
|
978
|
+
# update process-local variables
|
|
979
|
+
_peeler_process_context.ctx = PeelerProcessContext(
|
|
980
|
+
peeler, compute_residual, skip_features, chunk_length_samples, to_cpu
|
|
981
|
+
)
|
|
982
|
+
|
|
983
|
+
|
|
984
|
+
def _peeler_process_job(chunk_start_samples__n_resid_snips):
|
|
985
|
+
chunk_start_samples, n_resid_snips = chunk_start_samples__n_resid_snips
|
|
986
|
+
# by returning here, we are implicitly relying on pickle
|
|
987
|
+
# TODO: replace with manual np.saves
|
|
988
|
+
with torch.no_grad():
|
|
989
|
+
chunk_end_samples = None
|
|
990
|
+
chlen = _peeler_process_context.ctx.chunk_length_samples
|
|
991
|
+
if chlen is not None:
|
|
992
|
+
chunk_end_samples = chunk_start_samples + chlen
|
|
993
|
+
return _peeler_process_context.ctx.peeler.process_chunk(
|
|
994
|
+
chunk_start_samples,
|
|
995
|
+
n_resid_snips=n_resid_snips,
|
|
996
|
+
chunk_end_samples=chunk_end_samples,
|
|
997
|
+
return_residual=_peeler_process_context.ctx.compute_residual,
|
|
998
|
+
skip_features=_peeler_process_context.ctx.skip_features,
|
|
999
|
+
to_cpu=_peeler_process_context.ctx.to_cpu,
|
|
1000
|
+
)
|