DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
dartsort/peel/grab.py ADDED
@@ -0,0 +1,208 @@
1
+ from typing import Literal
2
+
3
+ import numpy as np
4
+ import torch
5
+ from spikeinterface import BaseRecording
6
+
7
+ from ..transform import WaveformPipeline
8
+ from ..util import spiketorch
9
+ from ..util.data_util import DARTsortSorting
10
+ from ..util.internal_config import (
11
+ FeaturizationConfig,
12
+ FitSamplingConfig,
13
+ WaveformConfig,
14
+ )
15
+ from ..util.waveform_util import make_channel_index
16
+ from .peel_base import (
17
+ BasePeeler,
18
+ PeelingBatchResult,
19
+ SpikeDataset,
20
+ peeling_empty_result,
21
+ )
22
+
23
+
24
+ class GrabAndFeaturize(BasePeeler):
25
+ """Extract and featurize raw waveform snippets at known times."""
26
+
27
+ peel_kind = "Grab and featurize"
28
+
29
+ def __init__(
30
+ self,
31
+ recording,
32
+ channel_index,
33
+ featurization_pipeline,
34
+ times_samples,
35
+ channels,
36
+ labels=None,
37
+ trough_offset_samples=42,
38
+ spike_length_samples=121,
39
+ chunk_length_samples=30_000,
40
+ n_waveforms_fit=20_000,
41
+ max_waveforms_fit=50_000,
42
+ fit_sampling: Literal["random", "amp_reweighted"] = "random",
43
+ n_seconds_fit=40,
44
+ fit_subsampling_random_state: int | np.random.Generator = 0,
45
+ dtype=torch.float,
46
+ ):
47
+ super().__init__(
48
+ recording=recording,
49
+ channel_index=channel_index,
50
+ featurization_pipeline=featurization_pipeline,
51
+ chunk_length_samples=chunk_length_samples,
52
+ chunk_margin_samples=max(
53
+ trough_offset_samples, spike_length_samples - trough_offset_samples
54
+ ),
55
+ trough_offset_samples=trough_offset_samples,
56
+ spike_length_samples=spike_length_samples,
57
+ n_seconds_fit=n_seconds_fit,
58
+ fit_subsampling_random_state=fit_subsampling_random_state,
59
+ n_waveforms_fit=n_waveforms_fit,
60
+ max_waveforms_fit=max_waveforms_fit,
61
+ fit_sampling=fit_sampling,
62
+ dtype=dtype,
63
+ )
64
+ self.register_buffer("times_samples", torch.asarray(times_samples))
65
+ self.register_buffer("channels", torch.asarray(channels))
66
+ if labels is not None:
67
+ self.register_buffer("labels", torch.asarray(labels))
68
+ else:
69
+ self.labels = None
70
+ assert self.times_samples.ndim == 1
71
+ assert self.times_samples.shape == self.channels.shape
72
+
73
+ def out_datasets(self):
74
+ datasets = super().out_datasets()
75
+ datasets.append(
76
+ SpikeDataset(name="indices", shape_per_spike=(), dtype=np.int64)
77
+ )
78
+ if self.labels is not None:
79
+ datasets.append(
80
+ SpikeDataset(name="labels", shape_per_spike=(), dtype=np.int64)
81
+ )
82
+ return datasets
83
+
84
+ def process_chunk(
85
+ self,
86
+ chunk_start_samples: int,
87
+ *,
88
+ chunk_end_samples: int | None = None,
89
+ return_residual: bool = False,
90
+ skip_features: bool = False,
91
+ n_resid_snips: int | None = None,
92
+ to_cpu: bool = True,
93
+ ) -> "PeelingBatchResult":
94
+ """Override process_chunk to skip empties."""
95
+ if chunk_end_samples is None:
96
+ chunk_end_samples = min(
97
+ self.recording.get_num_samples(),
98
+ chunk_start_samples + self.chunk_length_samples,
99
+ )
100
+ t_clip = self.b.times_samples.clip(chunk_start_samples, chunk_end_samples - 1)
101
+ in_chunk = self.b.times_samples == t_clip
102
+ if not in_chunk.any():
103
+ return peeling_empty_result
104
+
105
+ res = super().process_chunk(
106
+ chunk_start_samples,
107
+ n_resid_snips=n_resid_snips,
108
+ chunk_end_samples=chunk_end_samples,
109
+ return_residual=return_residual,
110
+ skip_features=skip_features,
111
+ to_cpu=to_cpu,
112
+ )
113
+ return res
114
+
115
+ @classmethod
116
+ def from_config(
117
+ cls,
118
+ recording: BaseRecording,
119
+ *,
120
+ sorting: DARTsortSorting,
121
+ waveform_cfg: WaveformConfig,
122
+ featurization_cfg: FeaturizationConfig,
123
+ sampling_cfg: FitSamplingConfig,
124
+ ):
125
+ geom = torch.tensor(recording.get_channel_locations())
126
+ channel_index = make_channel_index(
127
+ geom, featurization_cfg.extract_radius, to_torch=True
128
+ )
129
+ featurization_pipeline = WaveformPipeline.from_config(
130
+ geom=geom,
131
+ channel_index=channel_index,
132
+ featurization_cfg=featurization_cfg,
133
+ waveform_cfg=waveform_cfg,
134
+ sampling_frequency=recording.sampling_frequency,
135
+ )
136
+ trough_offset_samples = waveform_cfg.trough_offset_samples(
137
+ recording.sampling_frequency
138
+ )
139
+ spike_length_samples = waveform_cfg.spike_length_samples(
140
+ recording.sampling_frequency
141
+ )
142
+ return cls(
143
+ recording=recording,
144
+ channel_index=channel_index,
145
+ featurization_pipeline=featurization_pipeline,
146
+ times_samples=sorting.times_samples,
147
+ channels=sorting.channels,
148
+ labels=sorting.labels,
149
+ trough_offset_samples=trough_offset_samples,
150
+ spike_length_samples=spike_length_samples,
151
+ chunk_length_samples=30_000,
152
+ n_seconds_fit=100,
153
+ dtype=torch.float,
154
+ n_waveforms_fit=sampling_cfg.n_waveforms_fit,
155
+ max_waveforms_fit=sampling_cfg.max_waveforms_fit,
156
+ fit_subsampling_random_state=sampling_cfg.fit_subsampling_random_state,
157
+ fit_sampling=sampling_cfg.fit_sampling,
158
+ )
159
+
160
+ def peel_chunk(
161
+ self,
162
+ traces,
163
+ *,
164
+ chunk_start_samples=0,
165
+ left_margin=0,
166
+ right_margin=0,
167
+ return_residual=False,
168
+ return_waveforms=True,
169
+ ) -> PeelingBatchResult:
170
+ assert not return_residual
171
+
172
+ max_t = chunk_start_samples + self.chunk_length_samples - 1
173
+ in_chunk = self.b.times_samples == self.b.times_samples.clip(
174
+ chunk_start_samples, max_t
175
+ )
176
+ (in_chunk,) = in_chunk.nonzero(as_tuple=True)
177
+
178
+ if not in_chunk.numel():
179
+ return peeling_empty_result
180
+
181
+ chunk_left = chunk_start_samples - left_margin
182
+ times_rel = self.b.times_samples[in_chunk] - chunk_left
183
+ channels = self.b.channels[in_chunk]
184
+
185
+ assert times_rel.ndim == 1
186
+ assert channels.ndim == 1
187
+
188
+ n_spikes = in_chunk.numel()
189
+ res = PeelingBatchResult(
190
+ n_spikes=n_spikes,
191
+ indices=in_chunk,
192
+ times_samples=self.b.times_samples[in_chunk],
193
+ channels=channels,
194
+ )
195
+ if return_waveforms:
196
+ res["collisioncleaned_waveforms"] = spiketorch.grab_spikes(
197
+ traces,
198
+ times_rel,
199
+ channels,
200
+ self.channel_index,
201
+ trough_offset=self.trough_offset_samples,
202
+ spike_length_samples=self.spike_length_samples,
203
+ already_padded=False,
204
+ pad_value=torch.nan,
205
+ )
206
+ if self.labels is not None:
207
+ res["labels"] = self.labels[in_chunk]
208
+ return res