DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
import tempfile
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
from spikeinterface.core import BaseRecording, BaseRecordingSegment, read_binary_folder
|
|
6
|
+
from spikeinterface.core.recording_tools import get_chunk_with_margin
|
|
7
|
+
from spikeinterface.preprocessing.basepreprocessor import (
|
|
8
|
+
BasePreprocessor,
|
|
9
|
+
BasePreprocessorSegment,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
from ..util.noise_util import StationaryFactorizedNoise
|
|
13
|
+
from .simlib import default_temporal_kernel_npy, rbf_kernel_sqrt, generate_geom
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def get_background_recording(
|
|
17
|
+
noise_recording_folder: Path | None,
|
|
18
|
+
duration_samples,
|
|
19
|
+
geom=None,
|
|
20
|
+
probe_kwargs=None,
|
|
21
|
+
noise_kind="stationary_factorized_rbf",
|
|
22
|
+
noise_spatial_kernel_bandwidth=15.0,
|
|
23
|
+
noise_temporal_kernel: np.ndarray | str | Path = default_temporal_kernel_npy,
|
|
24
|
+
noise_fft_t=121,
|
|
25
|
+
random_seed=0,
|
|
26
|
+
dtype="float16",
|
|
27
|
+
sampling_frequency=30_000.0,
|
|
28
|
+
white_noise_scale=1.0,
|
|
29
|
+
n_jobs=1,
|
|
30
|
+
in_memory=False,
|
|
31
|
+
overwrite=False,
|
|
32
|
+
):
|
|
33
|
+
if not overwrite and noise_recording_folder is not None:
|
|
34
|
+
if (noise_recording_folder / "binary.json").exists():
|
|
35
|
+
return read_binary_folder(noise_recording_folder)
|
|
36
|
+
|
|
37
|
+
if geom is None:
|
|
38
|
+
geom = generate_geom(**probe_kwargs or {})
|
|
39
|
+
n_channels = len(geom)
|
|
40
|
+
scale = 0.0 if noise_kind == "zero" else white_noise_scale
|
|
41
|
+
recording = WhiteNoiseRecording(
|
|
42
|
+
duration_samples,
|
|
43
|
+
n_channels,
|
|
44
|
+
sampling_frequency,
|
|
45
|
+
scale=scale,
|
|
46
|
+
random_seed=random_seed,
|
|
47
|
+
dtype=dtype,
|
|
48
|
+
)
|
|
49
|
+
recording.set_dummy_probe_from_locations(geom)
|
|
50
|
+
|
|
51
|
+
if noise_kind == "zero":
|
|
52
|
+
return recording
|
|
53
|
+
|
|
54
|
+
if noise_kind == "white" and in_memory:
|
|
55
|
+
return recording.save_to_memory(n_jobs=1)
|
|
56
|
+
if noise_kind == "white":
|
|
57
|
+
assert noise_recording_folder is not None
|
|
58
|
+
return recording.save_to_folder(
|
|
59
|
+
noise_recording_folder, n_jobs=1, overwrite=overwrite # type: ignore
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
assert noise_kind == "stationary_factorized_rbf"
|
|
63
|
+
if not isinstance(noise_temporal_kernel, np.ndarray):
|
|
64
|
+
noise_temporal_kernel = np.load(noise_temporal_kernel)
|
|
65
|
+
assert isinstance(noise_temporal_kernel, np.ndarray)
|
|
66
|
+
assert noise_temporal_kernel.ndim == 1
|
|
67
|
+
block_size = len(noise_temporal_kernel) * 2 - 1
|
|
68
|
+
spatial_std, spatial_vt = rbf_kernel_sqrt(
|
|
69
|
+
geom, bandwidth=noise_spatial_kernel_bandwidth
|
|
70
|
+
)
|
|
71
|
+
noise_temporal_kernel = noise_temporal_kernel.astype(spatial_vt.dtype) + 0j
|
|
72
|
+
noise = StationaryFactorizedNoise(
|
|
73
|
+
spatial_std=spatial_std,
|
|
74
|
+
vt_spatial=spatial_vt,
|
|
75
|
+
kernel_fft=noise_temporal_kernel,
|
|
76
|
+
block_size=block_size,
|
|
77
|
+
t=noise_fft_t,
|
|
78
|
+
)
|
|
79
|
+
assert not in_memory, "Not implemented yet."
|
|
80
|
+
assert noise_recording_folder is not None
|
|
81
|
+
|
|
82
|
+
# white noise must be cached before convolving
|
|
83
|
+
with tempfile.TemporaryDirectory() as tdir:
|
|
84
|
+
recording = recording.save_to_folder(
|
|
85
|
+
Path(tdir) / "noiserecording", n_jobs=1 # type: ignore
|
|
86
|
+
)
|
|
87
|
+
recording = UnwhitenPreprocessor(noise, recording)
|
|
88
|
+
recording = recording.save_to_folder(
|
|
89
|
+
noise_recording_folder, # type: ignore
|
|
90
|
+
n_jobs=n_jobs,
|
|
91
|
+
pool_engine="thread",
|
|
92
|
+
overwrite=overwrite, # type: ignore
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
return recording
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class WhiteNoiseRecording(BaseRecording):
|
|
99
|
+
def __init__(
|
|
100
|
+
self,
|
|
101
|
+
duration_samples,
|
|
102
|
+
n_channels,
|
|
103
|
+
sampling_frequency,
|
|
104
|
+
scale=1.0,
|
|
105
|
+
n_segments=1,
|
|
106
|
+
random_seed=0,
|
|
107
|
+
dtype="float32",
|
|
108
|
+
channel_ids=None,
|
|
109
|
+
):
|
|
110
|
+
if channel_ids is None:
|
|
111
|
+
channel_ids = np.arange(n_channels)
|
|
112
|
+
else:
|
|
113
|
+
channel_ids = np.asarray(channel_ids)
|
|
114
|
+
assert channel_ids.shape == (n_channels,)
|
|
115
|
+
super().__init__(sampling_frequency, channel_ids=channel_ids, dtype=dtype) # type: ignore
|
|
116
|
+
self._serializability["json"] = False
|
|
117
|
+
self._serializability["pickle"] = False
|
|
118
|
+
segment = WhiteNoiseRecordingSegment(
|
|
119
|
+
duration_samples, n_channels, sampling_frequency, dtype, random_seed, scale
|
|
120
|
+
)
|
|
121
|
+
self.add_recording_segment(segment)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class WhiteNoiseRecordingSegment(BaseRecordingSegment):
|
|
125
|
+
def __init__(
|
|
126
|
+
self, duration_samples, n_channels, sampling_frequency, dtype, rg, scale
|
|
127
|
+
):
|
|
128
|
+
super().__init__(sampling_frequency=sampling_frequency)
|
|
129
|
+
self.duration_samples = duration_samples
|
|
130
|
+
self.chans_arange = np.arange(n_channels)
|
|
131
|
+
self.sampling_frequency = sampling_frequency
|
|
132
|
+
self.dtype = dtype
|
|
133
|
+
self.rg = np.random.default_rng(rg)
|
|
134
|
+
self.scale = scale
|
|
135
|
+
|
|
136
|
+
def get_num_samples(self):
|
|
137
|
+
return self.duration_samples
|
|
138
|
+
|
|
139
|
+
def get_traces(self, start_frame=None, end_frame=None, channel_indices=None):
|
|
140
|
+
start_frame = start_frame or 0
|
|
141
|
+
end_frame = self.get_num_samples() if end_frame is None else end_frame
|
|
142
|
+
channel_indices = slice(None) if channel_indices is None else channel_indices
|
|
143
|
+
n_frames = end_frame - start_frame
|
|
144
|
+
n_channels = self.chans_arange[channel_indices].size
|
|
145
|
+
if not self.scale:
|
|
146
|
+
traces = np.zeros((n_frames, n_channels), dtype=self.dtype)
|
|
147
|
+
else:
|
|
148
|
+
traces = self.rg.normal(scale=self.scale, size=(n_frames, n_channels))
|
|
149
|
+
return traces.astype(self.dtype, copy=False)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class UnwhitenPreprocessor(BasePreprocessor):
|
|
153
|
+
def __init__(self, noise, recording):
|
|
154
|
+
super().__init__(recording)
|
|
155
|
+
for parent_segment in recording._recording_segments:
|
|
156
|
+
self.add_recording_segment(
|
|
157
|
+
UnwhitenPreprocessorSegment(parent_segment, noise)
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class UnwhitenPreprocessorSegment(BasePreprocessorSegment):
|
|
162
|
+
def __init__(self, parent_recording_segment, noise):
|
|
163
|
+
super().__init__(parent_recording_segment)
|
|
164
|
+
self.noise = noise
|
|
165
|
+
|
|
166
|
+
def get_traces(self, start_frame, end_frame, channel_indices):
|
|
167
|
+
start_frame = start_frame or 0
|
|
168
|
+
end_frame = self.get_num_samples() if end_frame is None else end_frame
|
|
169
|
+
traces, lm, rm = get_chunk_with_margin(
|
|
170
|
+
self.parent_recording_segment,
|
|
171
|
+
start_frame,
|
|
172
|
+
end_frame,
|
|
173
|
+
margin=self.noise.margin,
|
|
174
|
+
add_reflect_padding=True,
|
|
175
|
+
channel_indices=None,
|
|
176
|
+
)
|
|
177
|
+
assert lm == rm == self.noise.margin
|
|
178
|
+
dtype = traces.dtype
|
|
179
|
+
traces = self.noise.unwhiten(traces).numpy(force=True)
|
|
180
|
+
traces = traces.astype(dtype, copy=False)
|
|
181
|
+
assert traces.shape[0] == end_frame - start_frame
|
|
182
|
+
if channel_indices is not None:
|
|
183
|
+
traces = traces[:, channel_indices]
|
|
184
|
+
return traces
|