DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,223 @@
1
+ from pathlib import Path
2
+ from typing import overload, Literal
3
+
4
+ import h5py
5
+ import numpy as np
6
+ import torch
7
+
8
+ from ..localize.localize_util import check_resume_or_overwrite, localize_hdf5
9
+ from .data_util import DARTsortSorting
10
+ from . import job_util
11
+ from .py_util import resolve_path
12
+ from ..peel.peel_base import BasePeeler
13
+ from .internal_config import FeaturizationConfig, ComputationConfig
14
+
15
+
16
+ @overload
17
+ def run_peeler(
18
+ peeler: BasePeeler,
19
+ *,
20
+ output_directory: str | Path,
21
+ hdf5_filename: str,
22
+ model_subdir: str,
23
+ featurization_cfg: FeaturizationConfig,
24
+ computation_cfg: ComputationConfig | None = None,
25
+ chunk_starts_samples: np.ndarray | None = None,
26
+ overwrite: bool = False,
27
+ residual_filename: str | Path | None = None,
28
+ show_progress: bool = True,
29
+ fit_only: Literal[True],
30
+ localization_dataset_name="point_source_localizations",
31
+ ) -> None: ...
32
+
33
+
34
+ @overload
35
+ def run_peeler(
36
+ peeler: BasePeeler,
37
+ *,
38
+ output_directory: str | Path,
39
+ hdf5_filename: str,
40
+ model_subdir: str,
41
+ featurization_cfg: FeaturizationConfig,
42
+ computation_cfg: ComputationConfig | None = None,
43
+ chunk_starts_samples: np.ndarray | None = None,
44
+ overwrite: bool = False,
45
+ residual_filename: str | Path | None = None,
46
+ show_progress: bool = True,
47
+ fit_only: Literal[False] = False,
48
+ localization_dataset_name="point_source_localizations",
49
+ ) -> DARTsortSorting: ...
50
+
51
+
52
+ @overload
53
+ def run_peeler(
54
+ peeler: BasePeeler,
55
+ *,
56
+ output_directory: str | Path,
57
+ hdf5_filename: str,
58
+ model_subdir: str,
59
+ featurization_cfg: FeaturizationConfig,
60
+ computation_cfg: ComputationConfig | None = None,
61
+ chunk_starts_samples: np.ndarray | None = None,
62
+ overwrite: bool = False,
63
+ residual_filename: str | Path | None = None,
64
+ show_progress: bool = True,
65
+ fit_only: bool = False,
66
+ localization_dataset_name="point_source_localizations",
67
+ ) -> DARTsortSorting | None: ...
68
+
69
+
70
+ def run_peeler(
71
+ peeler: BasePeeler,
72
+ *,
73
+ output_directory: str | Path,
74
+ hdf5_filename: str,
75
+ model_subdir: str,
76
+ featurization_cfg: FeaturizationConfig,
77
+ computation_cfg: ComputationConfig | None = None,
78
+ chunk_starts_samples: np.ndarray | None = None,
79
+ overwrite: bool = False,
80
+ residual_filename: str | Path | None = None,
81
+ show_progress: bool = True,
82
+ fit_only: bool = False,
83
+ localization_dataset_name="point_source_localizations",
84
+ ):
85
+ output_directory = resolve_path(output_directory)
86
+ output_directory.mkdir(exist_ok=True)
87
+ model_dir = output_directory / model_subdir
88
+ output_hdf5_filename = output_directory / hdf5_filename
89
+ if residual_filename is not None:
90
+ if not isinstance(residual_filename, Path):
91
+ residual_filename = output_directory / residual_filename
92
+ do_localization_later = (
93
+ not featurization_cfg.denoise_only
94
+ and featurization_cfg.do_localization
95
+ and not featurization_cfg.nn_localization
96
+ )
97
+ if computation_cfg is None:
98
+ computation_cfg = job_util.get_global_computation_config()
99
+
100
+ if peeler_is_done(
101
+ peeler,
102
+ output_hdf5_filename,
103
+ overwrite=overwrite,
104
+ chunk_starts_samples=chunk_starts_samples,
105
+ do_localization=do_localization_later,
106
+ localization_dataset_name=localization_dataset_name,
107
+ ):
108
+ return DARTsortSorting.from_peeling_hdf5(output_hdf5_filename)
109
+
110
+ # ensure torch linalg inits before launching threads...
111
+ _ensure_torch_linalg(computation_cfg)
112
+
113
+ # fit models if needed
114
+ peeler.load_or_fit_and_save_models(
115
+ model_dir, overwrite=overwrite, computation_cfg=computation_cfg
116
+ )
117
+ if fit_only:
118
+ return
119
+
120
+ # run main
121
+ n_resid_now = featurization_cfg.n_residual_snips * int(
122
+ not featurization_cfg.residual_later
123
+ )
124
+ peeler.peel(
125
+ output_hdf5_filename,
126
+ chunk_starts_samples=chunk_starts_samples,
127
+ overwrite=overwrite,
128
+ residual_filename=residual_filename,
129
+ show_progress=show_progress,
130
+ computation_cfg=computation_cfg,
131
+ total_residual_snips=n_resid_now,
132
+ stop_after_n_waveforms=featurization_cfg.stop_after_n,
133
+ shuffle=featurization_cfg.shuffle,
134
+ )
135
+
136
+ if featurization_cfg.residual_later:
137
+ peeler.run_subsampled_peeling(
138
+ output_hdf5_filename,
139
+ chunk_length_samples=peeler.spike_length_samples,
140
+ residual_to_h5=True,
141
+ skip_features=True,
142
+ ignore_resuming=True,
143
+ computation_cfg=computation_cfg,
144
+ n_chunks=featurization_cfg.n_residual_snips,
145
+ task_name="Residual snips",
146
+ overwrite=False,
147
+ ordered=True,
148
+ skip_last=True,
149
+ )
150
+
151
+ # do localization
152
+ if do_localization_later:
153
+ wf_name = featurization_cfg.output_waveforms_name
154
+ loc_amp_type = featurization_cfg.localization_amplitude_type
155
+ localize_hdf5(
156
+ output_hdf5_filename,
157
+ radius=featurization_cfg.localization_radius,
158
+ amplitude_vectors_dataset_name=f"{wf_name}_{loc_amp_type}_amplitude_vectors",
159
+ output_dataset_name=localization_dataset_name,
160
+ show_progress=show_progress,
161
+ n_jobs=computation_cfg.actual_n_jobs(),
162
+ device=computation_cfg.actual_device(),
163
+ localization_model=featurization_cfg.localization_model,
164
+ )
165
+
166
+ return DARTsortSorting.from_peeling_hdf5(output_hdf5_filename)
167
+
168
+
169
+ def peeler_is_done(
170
+ peeler,
171
+ output_hdf5_filename,
172
+ overwrite=False,
173
+ n_residual_snips=0,
174
+ chunk_starts_samples=None,
175
+ do_localization=True,
176
+ localization_dataset_name="point_source_localizations",
177
+ main_channels_dataset_name="channels",
178
+ ):
179
+ if overwrite:
180
+ return False
181
+
182
+ if not output_hdf5_filename.exists():
183
+ return False
184
+
185
+ if n_residual_snips:
186
+ with h5py.File(output_hdf5_filename, "r") as h5:
187
+ if "residual" not in h5:
188
+ return False
189
+ if len(h5["residual"]) < n_residual_snips: # type: ignore
190
+ return False
191
+
192
+ if do_localization:
193
+ (
194
+ done,
195
+ output_hdf5_filename,
196
+ next_batch_start,
197
+ ) = check_resume_or_overwrite(
198
+ output_hdf5_filename,
199
+ localization_dataset_name,
200
+ main_channels_dataset_name,
201
+ overwrite=False,
202
+ )
203
+ return done
204
+
205
+ last_chunk_start, n_residual_snips = peeler.check_resuming(
206
+ output_hdf5_filename,
207
+ overwrite=False,
208
+ )
209
+ chunk_starts_samples = peeler.get_chunk_starts(
210
+ chunk_starts_samples=chunk_starts_samples
211
+ )
212
+ return last_chunk_start >= max(chunk_starts_samples)
213
+
214
+
215
+ def _ensure_torch_linalg(computation_cfg):
216
+ device = computation_cfg.actual_device()
217
+ if device.type == "cpu":
218
+ return
219
+ if computation_cfg.actual_n_jobs() == 0:
220
+ return
221
+
222
+ torch.cuda.synchronize()
223
+ torch.inverse(torch.ones((0, 0), device=device))
@@ -0,0 +1,213 @@
1
+ import contextlib
2
+ import dataclasses
3
+ import json
4
+ import math
5
+ import os
6
+ import shutil
7
+ import signal
8
+ import subprocess
9
+ import sys
10
+ import threading
11
+ import time
12
+ from importlib.metadata import Distribution
13
+ from importlib.resources.abc import Traversable
14
+ from pathlib import Path
15
+ from typing import dataclass_transform
16
+
17
+ from pydantic import ConfigDict
18
+ from pydantic.dataclasses import dataclass as pydantic_dataclass
19
+
20
+ from .logging_util import DARTSORTVERBOSE, get_logger
21
+
22
+ logger = get_logger(__name__)
23
+
24
+ # check if we are installed in editable mode
25
+ pkgname = sys.modules[__name__].__name__.split(".")[0]
26
+ durl = Distribution.from_name(pkgname).read_text("direct_url.json")
27
+ assert durl is not None
28
+ EDITABLE = json.loads(durl).get("dir_info", {}).get("editable", False)
29
+
30
+
31
+ # without this, pydantic allows unknown keys, so you can easily
32
+ # make a typo in your parameter name!
33
+ pydantic_strict_cfg = ConfigDict(strict=True, extra="forbid")
34
+
35
+
36
+ # used for configuration objects in [internal_]config.py
37
+ @dataclass_transform(kw_only_default=True, frozen_default=True)
38
+ def cfg_dataclass(*args, frozen=True, kw_only=True, **kwargs):
39
+ return pydantic_dataclass(
40
+ *args, **kwargs, frozen=frozen, kw_only=kw_only, config=pydantic_strict_cfg
41
+ )
42
+
43
+
44
+ # lightweight dataclass defaults
45
+ @dataclass_transform(kw_only_default=True, eq_default=False)
46
+ def databag(*args, slots=True, kw_only=True, eq=False, repr=False, **kwargs):
47
+ return dataclasses.dataclass(
48
+ *args, **kwargs, slots=slots, kw_only=kw_only, eq=eq, repr=repr
49
+ )
50
+
51
+
52
+ # random utility classes
53
+
54
+
55
+ class timer:
56
+ """
57
+ with timer("hi"):
58
+ bubblesort(np.arange(1e6)[::-1])
59
+ # prints: hi took <> s
60
+ with timer("zoom") as tic:
61
+ pass
62
+ assert np.isclose(tic.dt, 0)
63
+ """
64
+
65
+ def __init__(self, name="timer", loglevel=DARTSORTVERBOSE):
66
+ self.loglevel = loglevel
67
+ self.name = name
68
+
69
+ def __enter__(self):
70
+ self.start = time.time()
71
+ return self
72
+
73
+ def __exit__(self, *args):
74
+ self.dt = time.time() - self.start
75
+ logger.log(self.loglevel, "%s took %ss", self.name, self.dt)
76
+
77
+
78
+ class NoKeyboardInterrupt:
79
+ """A context manager that we use to avoid ending up in invalid states."""
80
+
81
+ def handler(self, *sig):
82
+ if self.sig:
83
+ signal.signal(signal.SIGINT, self.old_handler)
84
+ sig, self.sig = self.sig, None
85
+ self.old_handler(*sig) # type: ignore
86
+ self.sig = sig
87
+
88
+ def __enter__(self):
89
+ # TODO: maybe should just handle this in the peeling code. would need to detect
90
+ # partially saved batches somehow... uhh... maybe based on the last sample saved,
91
+ # if we update that last and resume from there...
92
+ if threading.current_thread() is threading.main_thread() and os.name == "posix":
93
+ self.old_handler = signal.signal(signal.SIGINT, self.handler)
94
+ self.sig = None
95
+
96
+ def __exit__(self, type, value, traceback):
97
+ if threading.current_thread() is threading.main_thread() and os.name == "posix":
98
+ signal.signal(signal.SIGINT, self.old_handler)
99
+ if self.sig:
100
+ self.old_handler(*self.sig) # type: ignore
101
+
102
+
103
+ if threading.current_thread() is threading.main_thread() and os.name == "posix":
104
+ delay_keyboard_interrupt = NoKeyboardInterrupt()
105
+ else:
106
+ delay_keyboard_interrupt = contextlib.nullcontext()
107
+
108
+
109
+ # helper functions used as `type`s in argparse
110
+
111
+
112
+ def int_or_inf(s):
113
+ s = float(s)
114
+ if math.isfinite(s):
115
+ return int(s)
116
+ return s
117
+
118
+
119
+ def int_or_float(s):
120
+ s = s.strip()
121
+ if not s.strip("0123456789"):
122
+ return int(s)
123
+ return float(s)
124
+
125
+
126
+ def int_or_float_or_none(s):
127
+ s = s.strip()
128
+ if s.lower() in ("none", ""):
129
+ return None
130
+ if not s.strip("0123456789"):
131
+ return int(s)
132
+ return float(s)
133
+
134
+
135
+ def float_or_none(s):
136
+ s = s.strip()
137
+ if s.lower() in ("none", ""):
138
+ return None
139
+ return float(s)
140
+
141
+
142
+ def int_or_none(s):
143
+ s = s.strip()
144
+ if s.lower() in ("none", ""):
145
+ return None
146
+ return int(s)
147
+
148
+
149
+ def float_or_str(s):
150
+ try:
151
+ return float(s)
152
+ except ValueError:
153
+ return s
154
+
155
+
156
+ def str_or_none(s):
157
+ if s.lower() == "none":
158
+ return None
159
+ return s
160
+
161
+
162
+ # files and paths
163
+
164
+
165
+ def resolve_path(p: str | Path | Traversable | None, strict=False) -> Path:
166
+ if p is None:
167
+ raise ValueError("Can't resolve path None.")
168
+ if isinstance(p, Traversable):
169
+ assert isinstance(p, Path)
170
+ p = Path(p)
171
+ p = p.expanduser()
172
+ p = p.absolute()
173
+ p = p.resolve(strict=strict)
174
+ return p
175
+
176
+
177
+ def dartcopy2(icfg, src, dest):
178
+ if icfg.workdir_copier == "shutil":
179
+ try:
180
+ shutil.copy2(src, dest, follow_symlinks=icfg.workdir_follow_symlinks)
181
+ except shutil.SameFileError:
182
+ # this happens in a symlink workflow that I use sometimes
183
+ return
184
+ elif icfg.workdir_copier == "rsync":
185
+ _rsync(src, dest, archive=False, follow_symlinks=icfg.workdir_follow_symlinks)
186
+ else:
187
+ assert False
188
+
189
+
190
+ def dartcopytree(icfg, src, dest):
191
+ if icfg.workdir_copier == "shutil":
192
+ shutil.copytree(
193
+ src,
194
+ dest,
195
+ symlinks=not icfg.workdir_follow_symlinks,
196
+ dirs_exist_ok=True,
197
+ )
198
+ elif icfg.workdir_copier == "rsync":
199
+ _rsync(
200
+ f"{src}/",
201
+ f"{dest}/",
202
+ archive=True,
203
+ follow_symlinks=icfg.workdir_follow_symlinks,
204
+ )
205
+ else:
206
+ assert False
207
+
208
+
209
+ def _rsync(src, dest, archive=True, follow_symlinks=False):
210
+ archive_flags = ["-a"] if archive else []
211
+ link_flags = ["--no-links", "-L"] if follow_symlinks else []
212
+ res = subprocess.run(["rsync", *archive_flags, *link_flags, str(src), str(dest)])
213
+ assert not res.returncode
@@ -0,0 +1,128 @@
1
+ import pickle
2
+ from pathlib import Path
3
+
4
+ import numpy as np
5
+ import torch
6
+ from spikeinterface.core import Motion
7
+
8
+ try:
9
+ from dredge import dredge_ap
10
+
11
+ have_dredge = True
12
+ except ImportError:
13
+ have_dredge = False
14
+ dredge_ap = None
15
+ pass
16
+
17
+
18
+ def estimate_motion(
19
+ recording,
20
+ sorting,
21
+ output_directory=None,
22
+ filename="motion_est.pkl",
23
+ overwrite=False,
24
+ do_motion_estimation=True,
25
+ probe_boundary_padding_um=100.0,
26
+ spatial_bin_length_um: float = 1.0,
27
+ temporal_bin_length_s: float = 1.0,
28
+ window_step_um: float = 400.0,
29
+ window_scale_um: float = 450.0,
30
+ window_margin_um: float | None = None,
31
+ max_dt_s: float = 1000.0,
32
+ max_disp_um: float | None = None,
33
+ correlation_threshold: float = 0.1,
34
+ min_amplitude: float | None = None,
35
+ masked_correlation: bool = False,
36
+ weights_threshold: float = 0.2,
37
+ rigid: bool = False,
38
+ localizations_dataset_name="point_source_localizations",
39
+ amplitudes_dataset_name="denoised_ptp_amplitudes",
40
+ device: torch.device | None = None,
41
+ ):
42
+ if not do_motion_estimation:
43
+ return None
44
+
45
+ if output_directory is not None and not overwrite:
46
+ motion_est = try_load_motion_est(output_directory, filename)
47
+ if motion_est is not None:
48
+ return motion_est
49
+
50
+ if not have_dredge:
51
+ raise ValueError("Please install DREDge to use motion estimation.")
52
+
53
+ x = getattr(sorting, localizations_dataset_name)[:, 0]
54
+ z = getattr(sorting, localizations_dataset_name)[:, 2]
55
+ a = getattr(sorting, amplitudes_dataset_name)
56
+ geom = recording.get_channel_locations()
57
+ xmin = geom[:, 0].min() - probe_boundary_padding_um
58
+ xmax = geom[:, 0].max() + probe_boundary_padding_um
59
+ zmin = geom[:, 1].min() - probe_boundary_padding_um
60
+ zmax = geom[:, 1].max() + probe_boundary_padding_um
61
+ valid = x == x.clip(xmin, xmax)
62
+ valid &= z == z.clip(zmin, zmax)
63
+ if min_amplitude:
64
+ valid &= a >= min_amplitude
65
+ valid = np.flatnonzero(valid)
66
+
67
+ # features for registration
68
+ z = z[valid]
69
+ t_s = sorting.times_seconds[valid]
70
+ a = a[valid]
71
+
72
+ # run registration
73
+ assert have_dredge
74
+ assert dredge_ap is not None
75
+ motion_est, info = dredge_ap.register(
76
+ amps=a,
77
+ depths_um=z,
78
+ times_s=t_s,
79
+ rigid=rigid,
80
+ bin_um=spatial_bin_length_um,
81
+ bin_s=temporal_bin_length_s,
82
+ win_step_um=window_step_um, # type: ignore
83
+ weights_threshold_low=weights_threshold,
84
+ weights_threshold_high=weights_threshold,
85
+ win_scale_um=window_scale_um, # type: ignore
86
+ win_margin_um=window_margin_um,
87
+ count_masked_correlation=masked_correlation,
88
+ max_disp_um=max_disp_um,
89
+ max_dt_s=max_dt_s, # type: ignore
90
+ mincorr=correlation_threshold,
91
+ device=device,
92
+ )
93
+
94
+ if output_directory is not None:
95
+ save_motion_est(motion_est, output_directory, filename)
96
+
97
+ return motion_est
98
+
99
+
100
+ def try_load_motion_est(output_directory: Path, filename="motion_est.pkl"):
101
+ filename = output_directory / filename
102
+ if filename.exists():
103
+ with open(filename, "rb") as jar:
104
+ return pickle.load(jar)
105
+ return None
106
+
107
+
108
+ def save_motion_est(motion_est, output_directory: Path, filename="motion_est.pkl", overwrite: bool = False):
109
+ filename = output_directory / filename
110
+ if filename.exists() and not overwrite:
111
+ return
112
+ with open(filename, "wb") as jar:
113
+ pickle.dump(motion_est, jar)
114
+
115
+
116
+ def dredge_to_si(motion_est) -> Motion:
117
+ disp = motion_est.displacement
118
+ t = motion_est.time_bin_centers_s
119
+ if disp.ndim == 1:
120
+ # rigid case
121
+ return Motion(
122
+ displacement=disp[:, None],
123
+ temporal_bins_s=t,
124
+ spatial_bins_um=np.zeros(1),
125
+ )
126
+ else:
127
+ # TODO
128
+ raise NotImplementedError
@@ -0,0 +1,30 @@
1
+ from spikeinterface.preprocessing.basepreprocessor import (
2
+ BasePreprocessor,
3
+ BasePreprocessorSegment,
4
+ )
5
+
6
+
7
+ class PeeledResidualRecording(BasePreprocessor):
8
+
9
+ def __init__(self, peeler):
10
+ # this does not handle segments at all!
11
+ super().__init__(peeler.recording, dtype=peeler.dtype)
12
+ assert peeler.recording.get_num_segments() == 1
13
+ self.add_recording_segment(PeeledResidualSegment(peeler))
14
+ self._kwargs = dict(peeler=peeler)
15
+
16
+
17
+ class PeeledResidualSegment(BasePreprocessorSegment):
18
+ def __init__(self, peeler):
19
+ self.peeler = peeler
20
+
21
+ def get_traces(self, start_frame, end_frame, channel_indices):
22
+ # right now the peeler dives into segment 0 in process_chunk
23
+ # i guess that's something to think about...
24
+ stuff = self.peeler.process_chunk(
25
+ chunk_start_samples=start_frame,
26
+ chunk_end_samples=end_frame,
27
+ return_residual=True,
28
+ skip_features=True,
29
+ )
30
+ return stuff["residual"].numpy(force=True)[:, channel_indices]
@@ -0,0 +1,81 @@
1
+ import torch
2
+ from scipy.sparse import coo_array
3
+
4
+ try:
5
+ import cupy as cp # type: ignore
6
+ from cupyx.scipy.sparse import coo_matrix as cupy_coo_matrix # type: ignore
7
+
8
+ except ImportError:
9
+ cp = None
10
+ cupy_coo_matrix = lambda a, shape: NotImplemented
11
+
12
+
13
+ def coo_to_scipy(coo_tensor):
14
+ data = coo_tensor.values().numpy(force=True)
15
+ coords = coo_tensor.indices().numpy(force=True)
16
+ return coo_array((data, coords), shape=coo_tensor.shape)
17
+
18
+
19
+ def coo_to_cupy(coo_tensor):
20
+ assert cp is not None
21
+ data = cp.asarray(coo_tensor.values())
22
+ iijj = cp.asarray(coo_tensor.indices())
23
+ return cupy_coo_matrix((data, iijj), shape=coo_tensor.shape)
24
+
25
+
26
+ # sparse kmeans helpers
27
+
28
+
29
+ def sparse_centroid_distsq(X, centroids, labels, centroid_mask, dbufs):
30
+ neighbors = centroid_mask[labels]
31
+ coo = neighbors.nonzero()
32
+ ii, cc = coo.T
33
+ nn = len(ii)
34
+
35
+ dbufx, dbufc = dbufs
36
+ dbufx = dbufx.resize_(nn, *dbufx.shape[1:])
37
+ dbufc = dbufc.resize_(nn, *dbufc.shape[1:])
38
+
39
+ torch.index_select(X, dim=0, index=ii, out=dbufx)
40
+ torch.index_select(centroids, dim=0, index=cc, out=dbufc)
41
+ dsq = dbufx.sub_(dbufc).square_().sum(dim=1)
42
+
43
+ distsq_coo = torch.sparse_coo_tensor(
44
+ coo.T, dsq, size=(len(X), len(centroids)), is_coalesced=True
45
+ )
46
+
47
+ return distsq_coo, (dbufx, dbufc)
48
+
49
+
50
+ def distsq_to_lik_coo(distsq_coo, sigmasq, log_proportions, in_place=False):
51
+ liks = distsq_coo
52
+ if not in_place:
53
+ liks = liks.clone()
54
+ liks.values().mul_(-0.5 / sigmasq).add_(log_proportions[liks.indices()[1]])
55
+ return liks
56
+
57
+
58
+ def logsumexp_coo(coo):
59
+ """Like torch.sparse.softmax, assumes non-explicit 0s are -inf
60
+
61
+ out should be initialized to -inf. Uses stable logsumexp trick.
62
+ """
63
+ v = coo.values()
64
+ i = coo.indices()[0]
65
+
66
+ # first, put the max values into out
67
+ max_values = v.new_full((coo.shape[0],), -torch.inf)
68
+ logsumexps = v.new_zeros((coo.shape[0],))
69
+ max_values.scatter_reduce_(dim=0, index=i, src=v, reduce="amax")
70
+
71
+ # now stable exponential
72
+ sv = max_values[i]
73
+ torch.subtract(v, sv, out=sv)
74
+ sv.exp_()
75
+
76
+ # add stable exps, take log, add back max vals
77
+ logsumexps.scatter_add_(dim=0, index=i, src=sv)
78
+ logsumexps.log_()
79
+ logsumexps += max_values
80
+
81
+ return logsumexps