DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
dartsort/util/spikeio.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"""Utilities for reading waveforms from recordings
|
|
2
|
+
|
|
3
|
+
If you want to read many waveforms and store them in a file, also
|
|
4
|
+
consider peel.GrabAndLocalize, which can do this and optionally
|
|
5
|
+
also featurize the waveforms
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from os import SEEK_SET
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def read_full_waveforms(
|
|
14
|
+
recording,
|
|
15
|
+
times_samples,
|
|
16
|
+
channel_subset=None,
|
|
17
|
+
trough_offset_samples=42,
|
|
18
|
+
spike_length_samples=121,
|
|
19
|
+
return_in_uV=False,
|
|
20
|
+
):
|
|
21
|
+
assert times_samples.ndim == 1
|
|
22
|
+
assert times_samples.size > 0
|
|
23
|
+
assert times_samples.dtype.kind == "i"
|
|
24
|
+
assert times_samples.max() <= recording.get_num_samples() - (
|
|
25
|
+
spike_length_samples - trough_offset_samples
|
|
26
|
+
)
|
|
27
|
+
n_channels = recording.get_num_channels()
|
|
28
|
+
n_spikes = times_samples.size
|
|
29
|
+
read_times = times_samples - trough_offset_samples
|
|
30
|
+
|
|
31
|
+
if not return_in_uV and recording.binary_compatible_with(
|
|
32
|
+
file_offset=0, time_axis=0, file_paths_length=1
|
|
33
|
+
):
|
|
34
|
+
# fast path. this is like 2x as fast as the read_traces for loop
|
|
35
|
+
# below, but requires a recording on disk in a nice format
|
|
36
|
+
binary_path = recording.get_binary_description()["file_paths"][0]
|
|
37
|
+
if channel_subset is None:
|
|
38
|
+
return _read_full_waveforms_binary(
|
|
39
|
+
binary_path,
|
|
40
|
+
read_times,
|
|
41
|
+
n_channels=n_channels,
|
|
42
|
+
dtype=recording.dtype,
|
|
43
|
+
spike_length_samples=spike_length_samples,
|
|
44
|
+
)
|
|
45
|
+
else:
|
|
46
|
+
return _read_subset_waveforms_binary(
|
|
47
|
+
binary_path,
|
|
48
|
+
times_samples,
|
|
49
|
+
n_channels,
|
|
50
|
+
dtype=recording.dtype,
|
|
51
|
+
load_channels=channel_subset,
|
|
52
|
+
trough_offset_samples=trough_offset_samples,
|
|
53
|
+
spike_length_samples=spike_length_samples,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
waveforms = np.empty(
|
|
57
|
+
(n_spikes, spike_length_samples, n_channels), dtype=recording.dtype
|
|
58
|
+
)
|
|
59
|
+
for i, t in enumerate(read_times):
|
|
60
|
+
waveforms[i] = recording.get_traces(
|
|
61
|
+
0,
|
|
62
|
+
start_frame=t,
|
|
63
|
+
end_frame=t + spike_length_samples,
|
|
64
|
+
return_in_uV=return_in_uV,
|
|
65
|
+
)
|
|
66
|
+
if channel_subset is not None:
|
|
67
|
+
waveforms = waveforms[:, :, channel_subset]
|
|
68
|
+
|
|
69
|
+
return waveforms
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def read_waveforms_channel_index(
|
|
73
|
+
recording,
|
|
74
|
+
times_samples,
|
|
75
|
+
channel_index,
|
|
76
|
+
main_channels,
|
|
77
|
+
trough_offset_samples=42,
|
|
78
|
+
spike_length_samples=121,
|
|
79
|
+
fill_value=np.nan,
|
|
80
|
+
):
|
|
81
|
+
assert times_samples.ndim == 1
|
|
82
|
+
assert times_samples.size > 0
|
|
83
|
+
assert times_samples.dtype.kind == "i"
|
|
84
|
+
assert times_samples.min() >= trough_offset_samples
|
|
85
|
+
assert times_samples.max() <= recording.get_num_samples() - (
|
|
86
|
+
spike_length_samples - trough_offset_samples
|
|
87
|
+
)
|
|
88
|
+
n_channels = recording.get_num_channels()
|
|
89
|
+
|
|
90
|
+
if recording.binary_compatible_with(
|
|
91
|
+
file_offset=0, time_axis=0, file_paths_length=1
|
|
92
|
+
):
|
|
93
|
+
# fast path. this is like 2x as fast as the read_traces for loop
|
|
94
|
+
# below, but requires a recording on disk in a nice format
|
|
95
|
+
binary_path = recording.get_binary_description()["file_paths"][0]
|
|
96
|
+
return _read_waveforms_binary_channel_index(
|
|
97
|
+
binary_path,
|
|
98
|
+
times_samples,
|
|
99
|
+
channel_index=channel_index,
|
|
100
|
+
main_channels=main_channels,
|
|
101
|
+
n_channels=n_channels,
|
|
102
|
+
dtype=recording.dtype,
|
|
103
|
+
trough_offset_samples=trough_offset_samples,
|
|
104
|
+
spike_length_samples=spike_length_samples,
|
|
105
|
+
fill_value=fill_value,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
n_spikes = times_samples.size
|
|
109
|
+
waveforms = np.full(
|
|
110
|
+
(n_spikes, spike_length_samples, channel_index.shape[1]),
|
|
111
|
+
fill_value,
|
|
112
|
+
dtype=recording.dtype,
|
|
113
|
+
)
|
|
114
|
+
read_times = times_samples - trough_offset_samples
|
|
115
|
+
|
|
116
|
+
for i, t in enumerate(read_times):
|
|
117
|
+
chans = channel_index[main_channels[i]]
|
|
118
|
+
good = chans < n_channels
|
|
119
|
+
load_channel_ids = recording.channel_ids[chans[good]]
|
|
120
|
+
waveforms[i, :, good] = recording.get_traces(
|
|
121
|
+
0,
|
|
122
|
+
start_frame=t,
|
|
123
|
+
end_frame=t + spike_length_samples,
|
|
124
|
+
channel_ids=load_channel_ids,
|
|
125
|
+
).T
|
|
126
|
+
|
|
127
|
+
return waveforms
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def read_single_channel_waveforms(
|
|
131
|
+
recording,
|
|
132
|
+
times_samples,
|
|
133
|
+
channels,
|
|
134
|
+
trough_offset_samples=42,
|
|
135
|
+
spike_length_samples=121,
|
|
136
|
+
fill_value=np.nan,
|
|
137
|
+
):
|
|
138
|
+
single_channel_index = np.arange(recording.get_num_channels())[:, None]
|
|
139
|
+
return read_waveforms_channel_index(
|
|
140
|
+
recording,
|
|
141
|
+
times_samples,
|
|
142
|
+
channel_index=single_channel_index,
|
|
143
|
+
main_channels=channels,
|
|
144
|
+
trough_offset_samples=trough_offset_samples,
|
|
145
|
+
spike_length_samples=spike_length_samples,
|
|
146
|
+
fill_value=fill_value,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _read_full_waveforms_binary(
|
|
151
|
+
binary_path,
|
|
152
|
+
read_times_samples,
|
|
153
|
+
n_channels,
|
|
154
|
+
dtype,
|
|
155
|
+
spike_length_samples=121,
|
|
156
|
+
):
|
|
157
|
+
n_spikes = read_times_samples.size
|
|
158
|
+
waveforms = np.empty((n_spikes, spike_length_samples, n_channels), dtype=dtype)
|
|
159
|
+
offsets = read_times_samples * np.dtype(dtype).itemsize * n_channels
|
|
160
|
+
with open(binary_path, "rb") as binary:
|
|
161
|
+
for i, offset in enumerate(offsets):
|
|
162
|
+
binary.seek(offset, SEEK_SET)
|
|
163
|
+
waveforms[i] = np.fromfile(
|
|
164
|
+
binary,
|
|
165
|
+
dtype=dtype,
|
|
166
|
+
count=spike_length_samples * n_channels,
|
|
167
|
+
).reshape(spike_length_samples, n_channels)
|
|
168
|
+
return waveforms
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _read_subset_waveforms_binary(
|
|
172
|
+
binary_path,
|
|
173
|
+
times_samples,
|
|
174
|
+
n_channels,
|
|
175
|
+
dtype,
|
|
176
|
+
load_channels,
|
|
177
|
+
trough_offset_samples=42,
|
|
178
|
+
spike_length_samples=121,
|
|
179
|
+
):
|
|
180
|
+
n_spikes = times_samples.size
|
|
181
|
+
waveforms = np.empty(
|
|
182
|
+
(n_spikes, spike_length_samples, load_channels.size), dtype=dtype
|
|
183
|
+
)
|
|
184
|
+
load_times = times_samples - trough_offset_samples
|
|
185
|
+
offsets = load_times * np.dtype(dtype).itemsize * n_channels
|
|
186
|
+
with open(binary_path, "rb") as binary:
|
|
187
|
+
for i, offset in enumerate(offsets):
|
|
188
|
+
binary.seek(offset, SEEK_SET)
|
|
189
|
+
waveforms[i] = np.fromfile(
|
|
190
|
+
binary,
|
|
191
|
+
dtype=dtype,
|
|
192
|
+
count=spike_length_samples * n_channels,
|
|
193
|
+
).reshape(spike_length_samples, n_channels)[:, load_channels]
|
|
194
|
+
return waveforms
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _read_waveforms_binary_channel_index(
|
|
198
|
+
binary_path,
|
|
199
|
+
times_samples,
|
|
200
|
+
n_channels,
|
|
201
|
+
dtype,
|
|
202
|
+
channel_index,
|
|
203
|
+
main_channels,
|
|
204
|
+
trough_offset_samples=42,
|
|
205
|
+
spike_length_samples=121,
|
|
206
|
+
fill_value=np.nan,
|
|
207
|
+
):
|
|
208
|
+
n_spikes = times_samples.size
|
|
209
|
+
waveforms = np.full(
|
|
210
|
+
(n_spikes, spike_length_samples, channel_index.shape[1]),
|
|
211
|
+
fill_value,
|
|
212
|
+
dtype=dtype,
|
|
213
|
+
)
|
|
214
|
+
load_times = times_samples - trough_offset_samples
|
|
215
|
+
offsets = load_times * np.dtype(dtype).itemsize * n_channels
|
|
216
|
+
with open(binary_path, "rb") as binary:
|
|
217
|
+
for i, offset in enumerate(offsets):
|
|
218
|
+
binary.seek(offset, SEEK_SET)
|
|
219
|
+
wf = np.fromfile(
|
|
220
|
+
binary,
|
|
221
|
+
dtype=dtype,
|
|
222
|
+
count=spike_length_samples * n_channels,
|
|
223
|
+
).reshape(spike_length_samples, n_channels)
|
|
224
|
+
chans = channel_index[main_channels[i]]
|
|
225
|
+
good = chans < n_channels
|
|
226
|
+
waveforms[i, :, good] = wf.T[chans[good]]
|
|
227
|
+
return waveforms
|