DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import overload, Literal
|
|
3
|
+
|
|
4
|
+
import h5py
|
|
5
|
+
import numpy as np
|
|
6
|
+
import torch
|
|
7
|
+
|
|
8
|
+
from ..localize.localize_util import check_resume_or_overwrite, localize_hdf5
|
|
9
|
+
from .data_util import DARTsortSorting
|
|
10
|
+
from . import job_util
|
|
11
|
+
from .py_util import resolve_path
|
|
12
|
+
from ..peel.peel_base import BasePeeler
|
|
13
|
+
from .internal_config import FeaturizationConfig, ComputationConfig
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@overload
|
|
17
|
+
def run_peeler(
|
|
18
|
+
peeler: BasePeeler,
|
|
19
|
+
*,
|
|
20
|
+
output_directory: str | Path,
|
|
21
|
+
hdf5_filename: str,
|
|
22
|
+
model_subdir: str,
|
|
23
|
+
featurization_cfg: FeaturizationConfig,
|
|
24
|
+
computation_cfg: ComputationConfig | None = None,
|
|
25
|
+
chunk_starts_samples: np.ndarray | None = None,
|
|
26
|
+
overwrite: bool = False,
|
|
27
|
+
residual_filename: str | Path | None = None,
|
|
28
|
+
show_progress: bool = True,
|
|
29
|
+
fit_only: Literal[True],
|
|
30
|
+
localization_dataset_name="point_source_localizations",
|
|
31
|
+
) -> None: ...
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@overload
|
|
35
|
+
def run_peeler(
|
|
36
|
+
peeler: BasePeeler,
|
|
37
|
+
*,
|
|
38
|
+
output_directory: str | Path,
|
|
39
|
+
hdf5_filename: str,
|
|
40
|
+
model_subdir: str,
|
|
41
|
+
featurization_cfg: FeaturizationConfig,
|
|
42
|
+
computation_cfg: ComputationConfig | None = None,
|
|
43
|
+
chunk_starts_samples: np.ndarray | None = None,
|
|
44
|
+
overwrite: bool = False,
|
|
45
|
+
residual_filename: str | Path | None = None,
|
|
46
|
+
show_progress: bool = True,
|
|
47
|
+
fit_only: Literal[False] = False,
|
|
48
|
+
localization_dataset_name="point_source_localizations",
|
|
49
|
+
) -> DARTsortSorting: ...
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@overload
|
|
53
|
+
def run_peeler(
|
|
54
|
+
peeler: BasePeeler,
|
|
55
|
+
*,
|
|
56
|
+
output_directory: str | Path,
|
|
57
|
+
hdf5_filename: str,
|
|
58
|
+
model_subdir: str,
|
|
59
|
+
featurization_cfg: FeaturizationConfig,
|
|
60
|
+
computation_cfg: ComputationConfig | None = None,
|
|
61
|
+
chunk_starts_samples: np.ndarray | None = None,
|
|
62
|
+
overwrite: bool = False,
|
|
63
|
+
residual_filename: str | Path | None = None,
|
|
64
|
+
show_progress: bool = True,
|
|
65
|
+
fit_only: bool = False,
|
|
66
|
+
localization_dataset_name="point_source_localizations",
|
|
67
|
+
) -> DARTsortSorting | None: ...
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def run_peeler(
|
|
71
|
+
peeler: BasePeeler,
|
|
72
|
+
*,
|
|
73
|
+
output_directory: str | Path,
|
|
74
|
+
hdf5_filename: str,
|
|
75
|
+
model_subdir: str,
|
|
76
|
+
featurization_cfg: FeaturizationConfig,
|
|
77
|
+
computation_cfg: ComputationConfig | None = None,
|
|
78
|
+
chunk_starts_samples: np.ndarray | None = None,
|
|
79
|
+
overwrite: bool = False,
|
|
80
|
+
residual_filename: str | Path | None = None,
|
|
81
|
+
show_progress: bool = True,
|
|
82
|
+
fit_only: bool = False,
|
|
83
|
+
localization_dataset_name="point_source_localizations",
|
|
84
|
+
):
|
|
85
|
+
output_directory = resolve_path(output_directory)
|
|
86
|
+
output_directory.mkdir(exist_ok=True)
|
|
87
|
+
model_dir = output_directory / model_subdir
|
|
88
|
+
output_hdf5_filename = output_directory / hdf5_filename
|
|
89
|
+
if residual_filename is not None:
|
|
90
|
+
if not isinstance(residual_filename, Path):
|
|
91
|
+
residual_filename = output_directory / residual_filename
|
|
92
|
+
do_localization_later = (
|
|
93
|
+
not featurization_cfg.denoise_only
|
|
94
|
+
and featurization_cfg.do_localization
|
|
95
|
+
and not featurization_cfg.nn_localization
|
|
96
|
+
)
|
|
97
|
+
if computation_cfg is None:
|
|
98
|
+
computation_cfg = job_util.get_global_computation_config()
|
|
99
|
+
|
|
100
|
+
if peeler_is_done(
|
|
101
|
+
peeler,
|
|
102
|
+
output_hdf5_filename,
|
|
103
|
+
overwrite=overwrite,
|
|
104
|
+
chunk_starts_samples=chunk_starts_samples,
|
|
105
|
+
do_localization=do_localization_later,
|
|
106
|
+
localization_dataset_name=localization_dataset_name,
|
|
107
|
+
):
|
|
108
|
+
return DARTsortSorting.from_peeling_hdf5(output_hdf5_filename)
|
|
109
|
+
|
|
110
|
+
# ensure torch linalg inits before launching threads...
|
|
111
|
+
_ensure_torch_linalg(computation_cfg)
|
|
112
|
+
|
|
113
|
+
# fit models if needed
|
|
114
|
+
peeler.load_or_fit_and_save_models(
|
|
115
|
+
model_dir, overwrite=overwrite, computation_cfg=computation_cfg
|
|
116
|
+
)
|
|
117
|
+
if fit_only:
|
|
118
|
+
return
|
|
119
|
+
|
|
120
|
+
# run main
|
|
121
|
+
n_resid_now = featurization_cfg.n_residual_snips * int(
|
|
122
|
+
not featurization_cfg.residual_later
|
|
123
|
+
)
|
|
124
|
+
peeler.peel(
|
|
125
|
+
output_hdf5_filename,
|
|
126
|
+
chunk_starts_samples=chunk_starts_samples,
|
|
127
|
+
overwrite=overwrite,
|
|
128
|
+
residual_filename=residual_filename,
|
|
129
|
+
show_progress=show_progress,
|
|
130
|
+
computation_cfg=computation_cfg,
|
|
131
|
+
total_residual_snips=n_resid_now,
|
|
132
|
+
stop_after_n_waveforms=featurization_cfg.stop_after_n,
|
|
133
|
+
shuffle=featurization_cfg.shuffle,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
if featurization_cfg.residual_later:
|
|
137
|
+
peeler.run_subsampled_peeling(
|
|
138
|
+
output_hdf5_filename,
|
|
139
|
+
chunk_length_samples=peeler.spike_length_samples,
|
|
140
|
+
residual_to_h5=True,
|
|
141
|
+
skip_features=True,
|
|
142
|
+
ignore_resuming=True,
|
|
143
|
+
computation_cfg=computation_cfg,
|
|
144
|
+
n_chunks=featurization_cfg.n_residual_snips,
|
|
145
|
+
task_name="Residual snips",
|
|
146
|
+
overwrite=False,
|
|
147
|
+
ordered=True,
|
|
148
|
+
skip_last=True,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
# do localization
|
|
152
|
+
if do_localization_later:
|
|
153
|
+
wf_name = featurization_cfg.output_waveforms_name
|
|
154
|
+
loc_amp_type = featurization_cfg.localization_amplitude_type
|
|
155
|
+
localize_hdf5(
|
|
156
|
+
output_hdf5_filename,
|
|
157
|
+
radius=featurization_cfg.localization_radius,
|
|
158
|
+
amplitude_vectors_dataset_name=f"{wf_name}_{loc_amp_type}_amplitude_vectors",
|
|
159
|
+
output_dataset_name=localization_dataset_name,
|
|
160
|
+
show_progress=show_progress,
|
|
161
|
+
n_jobs=computation_cfg.actual_n_jobs(),
|
|
162
|
+
device=computation_cfg.actual_device(),
|
|
163
|
+
localization_model=featurization_cfg.localization_model,
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
return DARTsortSorting.from_peeling_hdf5(output_hdf5_filename)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def peeler_is_done(
|
|
170
|
+
peeler,
|
|
171
|
+
output_hdf5_filename,
|
|
172
|
+
overwrite=False,
|
|
173
|
+
n_residual_snips=0,
|
|
174
|
+
chunk_starts_samples=None,
|
|
175
|
+
do_localization=True,
|
|
176
|
+
localization_dataset_name="point_source_localizations",
|
|
177
|
+
main_channels_dataset_name="channels",
|
|
178
|
+
):
|
|
179
|
+
if overwrite:
|
|
180
|
+
return False
|
|
181
|
+
|
|
182
|
+
if not output_hdf5_filename.exists():
|
|
183
|
+
return False
|
|
184
|
+
|
|
185
|
+
if n_residual_snips:
|
|
186
|
+
with h5py.File(output_hdf5_filename, "r") as h5:
|
|
187
|
+
if "residual" not in h5:
|
|
188
|
+
return False
|
|
189
|
+
if len(h5["residual"]) < n_residual_snips: # type: ignore
|
|
190
|
+
return False
|
|
191
|
+
|
|
192
|
+
if do_localization:
|
|
193
|
+
(
|
|
194
|
+
done,
|
|
195
|
+
output_hdf5_filename,
|
|
196
|
+
next_batch_start,
|
|
197
|
+
) = check_resume_or_overwrite(
|
|
198
|
+
output_hdf5_filename,
|
|
199
|
+
localization_dataset_name,
|
|
200
|
+
main_channels_dataset_name,
|
|
201
|
+
overwrite=False,
|
|
202
|
+
)
|
|
203
|
+
return done
|
|
204
|
+
|
|
205
|
+
last_chunk_start, n_residual_snips = peeler.check_resuming(
|
|
206
|
+
output_hdf5_filename,
|
|
207
|
+
overwrite=False,
|
|
208
|
+
)
|
|
209
|
+
chunk_starts_samples = peeler.get_chunk_starts(
|
|
210
|
+
chunk_starts_samples=chunk_starts_samples
|
|
211
|
+
)
|
|
212
|
+
return last_chunk_start >= max(chunk_starts_samples)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _ensure_torch_linalg(computation_cfg):
|
|
216
|
+
device = computation_cfg.actual_device()
|
|
217
|
+
if device.type == "cpu":
|
|
218
|
+
return
|
|
219
|
+
if computation_cfg.actual_n_jobs() == 0:
|
|
220
|
+
return
|
|
221
|
+
|
|
222
|
+
torch.cuda.synchronize()
|
|
223
|
+
torch.inverse(torch.ones((0, 0), device=device))
|
dartsort/util/py_util.py
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import contextlib
|
|
2
|
+
import dataclasses
|
|
3
|
+
import json
|
|
4
|
+
import math
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import signal
|
|
8
|
+
import subprocess
|
|
9
|
+
import sys
|
|
10
|
+
import threading
|
|
11
|
+
import time
|
|
12
|
+
from importlib.metadata import Distribution
|
|
13
|
+
from importlib.resources.abc import Traversable
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import dataclass_transform
|
|
16
|
+
|
|
17
|
+
from pydantic import ConfigDict
|
|
18
|
+
from pydantic.dataclasses import dataclass as pydantic_dataclass
|
|
19
|
+
|
|
20
|
+
from .logging_util import DARTSORTVERBOSE, get_logger
|
|
21
|
+
|
|
22
|
+
logger = get_logger(__name__)
|
|
23
|
+
|
|
24
|
+
# check if we are installed in editable mode
|
|
25
|
+
pkgname = sys.modules[__name__].__name__.split(".")[0]
|
|
26
|
+
durl = Distribution.from_name(pkgname).read_text("direct_url.json")
|
|
27
|
+
assert durl is not None
|
|
28
|
+
EDITABLE = json.loads(durl).get("dir_info", {}).get("editable", False)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# without this, pydantic allows unknown keys, so you can easily
|
|
32
|
+
# make a typo in your parameter name!
|
|
33
|
+
pydantic_strict_cfg = ConfigDict(strict=True, extra="forbid")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# used for configuration objects in [internal_]config.py
|
|
37
|
+
@dataclass_transform(kw_only_default=True, frozen_default=True)
|
|
38
|
+
def cfg_dataclass(*args, frozen=True, kw_only=True, **kwargs):
|
|
39
|
+
return pydantic_dataclass(
|
|
40
|
+
*args, **kwargs, frozen=frozen, kw_only=kw_only, config=pydantic_strict_cfg
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# lightweight dataclass defaults
|
|
45
|
+
@dataclass_transform(kw_only_default=True, eq_default=False)
|
|
46
|
+
def databag(*args, slots=True, kw_only=True, eq=False, repr=False, **kwargs):
|
|
47
|
+
return dataclasses.dataclass(
|
|
48
|
+
*args, **kwargs, slots=slots, kw_only=kw_only, eq=eq, repr=repr
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# random utility classes
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class timer:
|
|
56
|
+
"""
|
|
57
|
+
with timer("hi"):
|
|
58
|
+
bubblesort(np.arange(1e6)[::-1])
|
|
59
|
+
# prints: hi took <> s
|
|
60
|
+
with timer("zoom") as tic:
|
|
61
|
+
pass
|
|
62
|
+
assert np.isclose(tic.dt, 0)
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
def __init__(self, name="timer", loglevel=DARTSORTVERBOSE):
|
|
66
|
+
self.loglevel = loglevel
|
|
67
|
+
self.name = name
|
|
68
|
+
|
|
69
|
+
def __enter__(self):
|
|
70
|
+
self.start = time.time()
|
|
71
|
+
return self
|
|
72
|
+
|
|
73
|
+
def __exit__(self, *args):
|
|
74
|
+
self.dt = time.time() - self.start
|
|
75
|
+
logger.log(self.loglevel, "%s took %ss", self.name, self.dt)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class NoKeyboardInterrupt:
|
|
79
|
+
"""A context manager that we use to avoid ending up in invalid states."""
|
|
80
|
+
|
|
81
|
+
def handler(self, *sig):
|
|
82
|
+
if self.sig:
|
|
83
|
+
signal.signal(signal.SIGINT, self.old_handler)
|
|
84
|
+
sig, self.sig = self.sig, None
|
|
85
|
+
self.old_handler(*sig) # type: ignore
|
|
86
|
+
self.sig = sig
|
|
87
|
+
|
|
88
|
+
def __enter__(self):
|
|
89
|
+
# TODO: maybe should just handle this in the peeling code. would need to detect
|
|
90
|
+
# partially saved batches somehow... uhh... maybe based on the last sample saved,
|
|
91
|
+
# if we update that last and resume from there...
|
|
92
|
+
if threading.current_thread() is threading.main_thread() and os.name == "posix":
|
|
93
|
+
self.old_handler = signal.signal(signal.SIGINT, self.handler)
|
|
94
|
+
self.sig = None
|
|
95
|
+
|
|
96
|
+
def __exit__(self, type, value, traceback):
|
|
97
|
+
if threading.current_thread() is threading.main_thread() and os.name == "posix":
|
|
98
|
+
signal.signal(signal.SIGINT, self.old_handler)
|
|
99
|
+
if self.sig:
|
|
100
|
+
self.old_handler(*self.sig) # type: ignore
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
if threading.current_thread() is threading.main_thread() and os.name == "posix":
|
|
104
|
+
delay_keyboard_interrupt = NoKeyboardInterrupt()
|
|
105
|
+
else:
|
|
106
|
+
delay_keyboard_interrupt = contextlib.nullcontext()
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# helper functions used as `type`s in argparse
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def int_or_inf(s):
|
|
113
|
+
s = float(s)
|
|
114
|
+
if math.isfinite(s):
|
|
115
|
+
return int(s)
|
|
116
|
+
return s
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def int_or_float(s):
|
|
120
|
+
s = s.strip()
|
|
121
|
+
if not s.strip("0123456789"):
|
|
122
|
+
return int(s)
|
|
123
|
+
return float(s)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def int_or_float_or_none(s):
|
|
127
|
+
s = s.strip()
|
|
128
|
+
if s.lower() in ("none", ""):
|
|
129
|
+
return None
|
|
130
|
+
if not s.strip("0123456789"):
|
|
131
|
+
return int(s)
|
|
132
|
+
return float(s)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def float_or_none(s):
|
|
136
|
+
s = s.strip()
|
|
137
|
+
if s.lower() in ("none", ""):
|
|
138
|
+
return None
|
|
139
|
+
return float(s)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def int_or_none(s):
|
|
143
|
+
s = s.strip()
|
|
144
|
+
if s.lower() in ("none", ""):
|
|
145
|
+
return None
|
|
146
|
+
return int(s)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def float_or_str(s):
|
|
150
|
+
try:
|
|
151
|
+
return float(s)
|
|
152
|
+
except ValueError:
|
|
153
|
+
return s
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def str_or_none(s):
|
|
157
|
+
if s.lower() == "none":
|
|
158
|
+
return None
|
|
159
|
+
return s
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
# files and paths
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def resolve_path(p: str | Path | Traversable | None, strict=False) -> Path:
|
|
166
|
+
if p is None:
|
|
167
|
+
raise ValueError("Can't resolve path None.")
|
|
168
|
+
if isinstance(p, Traversable):
|
|
169
|
+
assert isinstance(p, Path)
|
|
170
|
+
p = Path(p)
|
|
171
|
+
p = p.expanduser()
|
|
172
|
+
p = p.absolute()
|
|
173
|
+
p = p.resolve(strict=strict)
|
|
174
|
+
return p
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def dartcopy2(icfg, src, dest):
|
|
178
|
+
if icfg.workdir_copier == "shutil":
|
|
179
|
+
try:
|
|
180
|
+
shutil.copy2(src, dest, follow_symlinks=icfg.workdir_follow_symlinks)
|
|
181
|
+
except shutil.SameFileError:
|
|
182
|
+
# this happens in a symlink workflow that I use sometimes
|
|
183
|
+
return
|
|
184
|
+
elif icfg.workdir_copier == "rsync":
|
|
185
|
+
_rsync(src, dest, archive=False, follow_symlinks=icfg.workdir_follow_symlinks)
|
|
186
|
+
else:
|
|
187
|
+
assert False
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def dartcopytree(icfg, src, dest):
|
|
191
|
+
if icfg.workdir_copier == "shutil":
|
|
192
|
+
shutil.copytree(
|
|
193
|
+
src,
|
|
194
|
+
dest,
|
|
195
|
+
symlinks=not icfg.workdir_follow_symlinks,
|
|
196
|
+
dirs_exist_ok=True,
|
|
197
|
+
)
|
|
198
|
+
elif icfg.workdir_copier == "rsync":
|
|
199
|
+
_rsync(
|
|
200
|
+
f"{src}/",
|
|
201
|
+
f"{dest}/",
|
|
202
|
+
archive=True,
|
|
203
|
+
follow_symlinks=icfg.workdir_follow_symlinks,
|
|
204
|
+
)
|
|
205
|
+
else:
|
|
206
|
+
assert False
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _rsync(src, dest, archive=True, follow_symlinks=False):
|
|
210
|
+
archive_flags = ["-a"] if archive else []
|
|
211
|
+
link_flags = ["--no-links", "-L"] if follow_symlinks else []
|
|
212
|
+
res = subprocess.run(["rsync", *archive_flags, *link_flags, str(src), str(dest)])
|
|
213
|
+
assert not res.returncode
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import pickle
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import torch
|
|
6
|
+
from spikeinterface.core import Motion
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
from dredge import dredge_ap
|
|
10
|
+
|
|
11
|
+
have_dredge = True
|
|
12
|
+
except ImportError:
|
|
13
|
+
have_dredge = False
|
|
14
|
+
dredge_ap = None
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def estimate_motion(
|
|
19
|
+
recording,
|
|
20
|
+
sorting,
|
|
21
|
+
output_directory=None,
|
|
22
|
+
filename="motion_est.pkl",
|
|
23
|
+
overwrite=False,
|
|
24
|
+
do_motion_estimation=True,
|
|
25
|
+
probe_boundary_padding_um=100.0,
|
|
26
|
+
spatial_bin_length_um: float = 1.0,
|
|
27
|
+
temporal_bin_length_s: float = 1.0,
|
|
28
|
+
window_step_um: float = 400.0,
|
|
29
|
+
window_scale_um: float = 450.0,
|
|
30
|
+
window_margin_um: float | None = None,
|
|
31
|
+
max_dt_s: float = 1000.0,
|
|
32
|
+
max_disp_um: float | None = None,
|
|
33
|
+
correlation_threshold: float = 0.1,
|
|
34
|
+
min_amplitude: float | None = None,
|
|
35
|
+
masked_correlation: bool = False,
|
|
36
|
+
weights_threshold: float = 0.2,
|
|
37
|
+
rigid: bool = False,
|
|
38
|
+
localizations_dataset_name="point_source_localizations",
|
|
39
|
+
amplitudes_dataset_name="denoised_ptp_amplitudes",
|
|
40
|
+
device: torch.device | None = None,
|
|
41
|
+
):
|
|
42
|
+
if not do_motion_estimation:
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
if output_directory is not None and not overwrite:
|
|
46
|
+
motion_est = try_load_motion_est(output_directory, filename)
|
|
47
|
+
if motion_est is not None:
|
|
48
|
+
return motion_est
|
|
49
|
+
|
|
50
|
+
if not have_dredge:
|
|
51
|
+
raise ValueError("Please install DREDge to use motion estimation.")
|
|
52
|
+
|
|
53
|
+
x = getattr(sorting, localizations_dataset_name)[:, 0]
|
|
54
|
+
z = getattr(sorting, localizations_dataset_name)[:, 2]
|
|
55
|
+
a = getattr(sorting, amplitudes_dataset_name)
|
|
56
|
+
geom = recording.get_channel_locations()
|
|
57
|
+
xmin = geom[:, 0].min() - probe_boundary_padding_um
|
|
58
|
+
xmax = geom[:, 0].max() + probe_boundary_padding_um
|
|
59
|
+
zmin = geom[:, 1].min() - probe_boundary_padding_um
|
|
60
|
+
zmax = geom[:, 1].max() + probe_boundary_padding_um
|
|
61
|
+
valid = x == x.clip(xmin, xmax)
|
|
62
|
+
valid &= z == z.clip(zmin, zmax)
|
|
63
|
+
if min_amplitude:
|
|
64
|
+
valid &= a >= min_amplitude
|
|
65
|
+
valid = np.flatnonzero(valid)
|
|
66
|
+
|
|
67
|
+
# features for registration
|
|
68
|
+
z = z[valid]
|
|
69
|
+
t_s = sorting.times_seconds[valid]
|
|
70
|
+
a = a[valid]
|
|
71
|
+
|
|
72
|
+
# run registration
|
|
73
|
+
assert have_dredge
|
|
74
|
+
assert dredge_ap is not None
|
|
75
|
+
motion_est, info = dredge_ap.register(
|
|
76
|
+
amps=a,
|
|
77
|
+
depths_um=z,
|
|
78
|
+
times_s=t_s,
|
|
79
|
+
rigid=rigid,
|
|
80
|
+
bin_um=spatial_bin_length_um,
|
|
81
|
+
bin_s=temporal_bin_length_s,
|
|
82
|
+
win_step_um=window_step_um, # type: ignore
|
|
83
|
+
weights_threshold_low=weights_threshold,
|
|
84
|
+
weights_threshold_high=weights_threshold,
|
|
85
|
+
win_scale_um=window_scale_um, # type: ignore
|
|
86
|
+
win_margin_um=window_margin_um,
|
|
87
|
+
count_masked_correlation=masked_correlation,
|
|
88
|
+
max_disp_um=max_disp_um,
|
|
89
|
+
max_dt_s=max_dt_s, # type: ignore
|
|
90
|
+
mincorr=correlation_threshold,
|
|
91
|
+
device=device,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
if output_directory is not None:
|
|
95
|
+
save_motion_est(motion_est, output_directory, filename)
|
|
96
|
+
|
|
97
|
+
return motion_est
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def try_load_motion_est(output_directory: Path, filename="motion_est.pkl"):
|
|
101
|
+
filename = output_directory / filename
|
|
102
|
+
if filename.exists():
|
|
103
|
+
with open(filename, "rb") as jar:
|
|
104
|
+
return pickle.load(jar)
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def save_motion_est(motion_est, output_directory: Path, filename="motion_est.pkl", overwrite: bool = False):
|
|
109
|
+
filename = output_directory / filename
|
|
110
|
+
if filename.exists() and not overwrite:
|
|
111
|
+
return
|
|
112
|
+
with open(filename, "wb") as jar:
|
|
113
|
+
pickle.dump(motion_est, jar)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def dredge_to_si(motion_est) -> Motion:
|
|
117
|
+
disp = motion_est.displacement
|
|
118
|
+
t = motion_est.time_bin_centers_s
|
|
119
|
+
if disp.ndim == 1:
|
|
120
|
+
# rigid case
|
|
121
|
+
return Motion(
|
|
122
|
+
displacement=disp[:, None],
|
|
123
|
+
temporal_bins_s=t,
|
|
124
|
+
spatial_bins_um=np.zeros(1),
|
|
125
|
+
)
|
|
126
|
+
else:
|
|
127
|
+
# TODO
|
|
128
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from spikeinterface.preprocessing.basepreprocessor import (
|
|
2
|
+
BasePreprocessor,
|
|
3
|
+
BasePreprocessorSegment,
|
|
4
|
+
)
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class PeeledResidualRecording(BasePreprocessor):
|
|
8
|
+
|
|
9
|
+
def __init__(self, peeler):
|
|
10
|
+
# this does not handle segments at all!
|
|
11
|
+
super().__init__(peeler.recording, dtype=peeler.dtype)
|
|
12
|
+
assert peeler.recording.get_num_segments() == 1
|
|
13
|
+
self.add_recording_segment(PeeledResidualSegment(peeler))
|
|
14
|
+
self._kwargs = dict(peeler=peeler)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class PeeledResidualSegment(BasePreprocessorSegment):
|
|
18
|
+
def __init__(self, peeler):
|
|
19
|
+
self.peeler = peeler
|
|
20
|
+
|
|
21
|
+
def get_traces(self, start_frame, end_frame, channel_indices):
|
|
22
|
+
# right now the peeler dives into segment 0 in process_chunk
|
|
23
|
+
# i guess that's something to think about...
|
|
24
|
+
stuff = self.peeler.process_chunk(
|
|
25
|
+
chunk_start_samples=start_frame,
|
|
26
|
+
chunk_end_samples=end_frame,
|
|
27
|
+
return_residual=True,
|
|
28
|
+
skip_features=True,
|
|
29
|
+
)
|
|
30
|
+
return stuff["residual"].numpy(force=True)[:, channel_indices]
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from scipy.sparse import coo_array
|
|
3
|
+
|
|
4
|
+
try:
|
|
5
|
+
import cupy as cp # type: ignore
|
|
6
|
+
from cupyx.scipy.sparse import coo_matrix as cupy_coo_matrix # type: ignore
|
|
7
|
+
|
|
8
|
+
except ImportError:
|
|
9
|
+
cp = None
|
|
10
|
+
cupy_coo_matrix = lambda a, shape: NotImplemented
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def coo_to_scipy(coo_tensor):
|
|
14
|
+
data = coo_tensor.values().numpy(force=True)
|
|
15
|
+
coords = coo_tensor.indices().numpy(force=True)
|
|
16
|
+
return coo_array((data, coords), shape=coo_tensor.shape)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def coo_to_cupy(coo_tensor):
|
|
20
|
+
assert cp is not None
|
|
21
|
+
data = cp.asarray(coo_tensor.values())
|
|
22
|
+
iijj = cp.asarray(coo_tensor.indices())
|
|
23
|
+
return cupy_coo_matrix((data, iijj), shape=coo_tensor.shape)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# sparse kmeans helpers
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def sparse_centroid_distsq(X, centroids, labels, centroid_mask, dbufs):
|
|
30
|
+
neighbors = centroid_mask[labels]
|
|
31
|
+
coo = neighbors.nonzero()
|
|
32
|
+
ii, cc = coo.T
|
|
33
|
+
nn = len(ii)
|
|
34
|
+
|
|
35
|
+
dbufx, dbufc = dbufs
|
|
36
|
+
dbufx = dbufx.resize_(nn, *dbufx.shape[1:])
|
|
37
|
+
dbufc = dbufc.resize_(nn, *dbufc.shape[1:])
|
|
38
|
+
|
|
39
|
+
torch.index_select(X, dim=0, index=ii, out=dbufx)
|
|
40
|
+
torch.index_select(centroids, dim=0, index=cc, out=dbufc)
|
|
41
|
+
dsq = dbufx.sub_(dbufc).square_().sum(dim=1)
|
|
42
|
+
|
|
43
|
+
distsq_coo = torch.sparse_coo_tensor(
|
|
44
|
+
coo.T, dsq, size=(len(X), len(centroids)), is_coalesced=True
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
return distsq_coo, (dbufx, dbufc)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def distsq_to_lik_coo(distsq_coo, sigmasq, log_proportions, in_place=False):
|
|
51
|
+
liks = distsq_coo
|
|
52
|
+
if not in_place:
|
|
53
|
+
liks = liks.clone()
|
|
54
|
+
liks.values().mul_(-0.5 / sigmasq).add_(log_proportions[liks.indices()[1]])
|
|
55
|
+
return liks
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def logsumexp_coo(coo):
|
|
59
|
+
"""Like torch.sparse.softmax, assumes non-explicit 0s are -inf
|
|
60
|
+
|
|
61
|
+
out should be initialized to -inf. Uses stable logsumexp trick.
|
|
62
|
+
"""
|
|
63
|
+
v = coo.values()
|
|
64
|
+
i = coo.indices()[0]
|
|
65
|
+
|
|
66
|
+
# first, put the max values into out
|
|
67
|
+
max_values = v.new_full((coo.shape[0],), -torch.inf)
|
|
68
|
+
logsumexps = v.new_zeros((coo.shape[0],))
|
|
69
|
+
max_values.scatter_reduce_(dim=0, index=i, src=v, reduce="amax")
|
|
70
|
+
|
|
71
|
+
# now stable exponential
|
|
72
|
+
sv = max_values[i]
|
|
73
|
+
torch.subtract(v, sv, out=sv)
|
|
74
|
+
sv.exp_()
|
|
75
|
+
|
|
76
|
+
# add stable exps, take log, add back max vals
|
|
77
|
+
logsumexps.scatter_add_(dim=0, index=i, src=sv)
|
|
78
|
+
logsumexps.log_()
|
|
79
|
+
logsumexps += max_values
|
|
80
|
+
|
|
81
|
+
return logsumexps
|