DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,184 @@
1
+ from pathlib import Path
2
+ import tempfile
3
+
4
+ import numpy as np
5
+ from spikeinterface.core import BaseRecording, BaseRecordingSegment, read_binary_folder
6
+ from spikeinterface.core.recording_tools import get_chunk_with_margin
7
+ from spikeinterface.preprocessing.basepreprocessor import (
8
+ BasePreprocessor,
9
+ BasePreprocessorSegment,
10
+ )
11
+
12
+ from ..util.noise_util import StationaryFactorizedNoise
13
+ from .simlib import default_temporal_kernel_npy, rbf_kernel_sqrt, generate_geom
14
+
15
+
16
+ def get_background_recording(
17
+ noise_recording_folder: Path | None,
18
+ duration_samples,
19
+ geom=None,
20
+ probe_kwargs=None,
21
+ noise_kind="stationary_factorized_rbf",
22
+ noise_spatial_kernel_bandwidth=15.0,
23
+ noise_temporal_kernel: np.ndarray | str | Path = default_temporal_kernel_npy,
24
+ noise_fft_t=121,
25
+ random_seed=0,
26
+ dtype="float16",
27
+ sampling_frequency=30_000.0,
28
+ white_noise_scale=1.0,
29
+ n_jobs=1,
30
+ in_memory=False,
31
+ overwrite=False,
32
+ ):
33
+ if not overwrite and noise_recording_folder is not None:
34
+ if (noise_recording_folder / "binary.json").exists():
35
+ return read_binary_folder(noise_recording_folder)
36
+
37
+ if geom is None:
38
+ geom = generate_geom(**probe_kwargs or {})
39
+ n_channels = len(geom)
40
+ scale = 0.0 if noise_kind == "zero" else white_noise_scale
41
+ recording = WhiteNoiseRecording(
42
+ duration_samples,
43
+ n_channels,
44
+ sampling_frequency,
45
+ scale=scale,
46
+ random_seed=random_seed,
47
+ dtype=dtype,
48
+ )
49
+ recording.set_dummy_probe_from_locations(geom)
50
+
51
+ if noise_kind == "zero":
52
+ return recording
53
+
54
+ if noise_kind == "white" and in_memory:
55
+ return recording.save_to_memory(n_jobs=1)
56
+ if noise_kind == "white":
57
+ assert noise_recording_folder is not None
58
+ return recording.save_to_folder(
59
+ noise_recording_folder, n_jobs=1, overwrite=overwrite # type: ignore
60
+ )
61
+
62
+ assert noise_kind == "stationary_factorized_rbf"
63
+ if not isinstance(noise_temporal_kernel, np.ndarray):
64
+ noise_temporal_kernel = np.load(noise_temporal_kernel)
65
+ assert isinstance(noise_temporal_kernel, np.ndarray)
66
+ assert noise_temporal_kernel.ndim == 1
67
+ block_size = len(noise_temporal_kernel) * 2 - 1
68
+ spatial_std, spatial_vt = rbf_kernel_sqrt(
69
+ geom, bandwidth=noise_spatial_kernel_bandwidth
70
+ )
71
+ noise_temporal_kernel = noise_temporal_kernel.astype(spatial_vt.dtype) + 0j
72
+ noise = StationaryFactorizedNoise(
73
+ spatial_std=spatial_std,
74
+ vt_spatial=spatial_vt,
75
+ kernel_fft=noise_temporal_kernel,
76
+ block_size=block_size,
77
+ t=noise_fft_t,
78
+ )
79
+ assert not in_memory, "Not implemented yet."
80
+ assert noise_recording_folder is not None
81
+
82
+ # white noise must be cached before convolving
83
+ with tempfile.TemporaryDirectory() as tdir:
84
+ recording = recording.save_to_folder(
85
+ Path(tdir) / "noiserecording", n_jobs=1 # type: ignore
86
+ )
87
+ recording = UnwhitenPreprocessor(noise, recording)
88
+ recording = recording.save_to_folder(
89
+ noise_recording_folder, # type: ignore
90
+ n_jobs=n_jobs,
91
+ pool_engine="thread",
92
+ overwrite=overwrite, # type: ignore
93
+ )
94
+
95
+ return recording
96
+
97
+
98
+ class WhiteNoiseRecording(BaseRecording):
99
+ def __init__(
100
+ self,
101
+ duration_samples,
102
+ n_channels,
103
+ sampling_frequency,
104
+ scale=1.0,
105
+ n_segments=1,
106
+ random_seed=0,
107
+ dtype="float32",
108
+ channel_ids=None,
109
+ ):
110
+ if channel_ids is None:
111
+ channel_ids = np.arange(n_channels)
112
+ else:
113
+ channel_ids = np.asarray(channel_ids)
114
+ assert channel_ids.shape == (n_channels,)
115
+ super().__init__(sampling_frequency, channel_ids=channel_ids, dtype=dtype) # type: ignore
116
+ self._serializability["json"] = False
117
+ self._serializability["pickle"] = False
118
+ segment = WhiteNoiseRecordingSegment(
119
+ duration_samples, n_channels, sampling_frequency, dtype, random_seed, scale
120
+ )
121
+ self.add_recording_segment(segment)
122
+
123
+
124
+ class WhiteNoiseRecordingSegment(BaseRecordingSegment):
125
+ def __init__(
126
+ self, duration_samples, n_channels, sampling_frequency, dtype, rg, scale
127
+ ):
128
+ super().__init__(sampling_frequency=sampling_frequency)
129
+ self.duration_samples = duration_samples
130
+ self.chans_arange = np.arange(n_channels)
131
+ self.sampling_frequency = sampling_frequency
132
+ self.dtype = dtype
133
+ self.rg = np.random.default_rng(rg)
134
+ self.scale = scale
135
+
136
+ def get_num_samples(self):
137
+ return self.duration_samples
138
+
139
+ def get_traces(self, start_frame=None, end_frame=None, channel_indices=None):
140
+ start_frame = start_frame or 0
141
+ end_frame = self.get_num_samples() if end_frame is None else end_frame
142
+ channel_indices = slice(None) if channel_indices is None else channel_indices
143
+ n_frames = end_frame - start_frame
144
+ n_channels = self.chans_arange[channel_indices].size
145
+ if not self.scale:
146
+ traces = np.zeros((n_frames, n_channels), dtype=self.dtype)
147
+ else:
148
+ traces = self.rg.normal(scale=self.scale, size=(n_frames, n_channels))
149
+ return traces.astype(self.dtype, copy=False)
150
+
151
+
152
+ class UnwhitenPreprocessor(BasePreprocessor):
153
+ def __init__(self, noise, recording):
154
+ super().__init__(recording)
155
+ for parent_segment in recording._recording_segments:
156
+ self.add_recording_segment(
157
+ UnwhitenPreprocessorSegment(parent_segment, noise)
158
+ )
159
+
160
+
161
+ class UnwhitenPreprocessorSegment(BasePreprocessorSegment):
162
+ def __init__(self, parent_recording_segment, noise):
163
+ super().__init__(parent_recording_segment)
164
+ self.noise = noise
165
+
166
+ def get_traces(self, start_frame, end_frame, channel_indices):
167
+ start_frame = start_frame or 0
168
+ end_frame = self.get_num_samples() if end_frame is None else end_frame
169
+ traces, lm, rm = get_chunk_with_margin(
170
+ self.parent_recording_segment,
171
+ start_frame,
172
+ end_frame,
173
+ margin=self.noise.margin,
174
+ add_reflect_padding=True,
175
+ channel_indices=None,
176
+ )
177
+ assert lm == rm == self.noise.margin
178
+ dtype = traces.dtype
179
+ traces = self.noise.unwhiten(traces).numpy(force=True)
180
+ traces = traces.astype(dtype, copy=False)
181
+ assert traces.shape[0] == end_frame - start_frame
182
+ if channel_indices is not None:
183
+ traces = traces[:, channel_indices]
184
+ return traces