DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,227 @@
1
+ """Utilities for reading waveforms from recordings
2
+
3
+ If you want to read many waveforms and store them in a file, also
4
+ consider peel.GrabAndLocalize, which can do this and optionally
5
+ also featurize the waveforms
6
+ """
7
+
8
+ from os import SEEK_SET
9
+
10
+ import numpy as np
11
+
12
+
13
+ def read_full_waveforms(
14
+ recording,
15
+ times_samples,
16
+ channel_subset=None,
17
+ trough_offset_samples=42,
18
+ spike_length_samples=121,
19
+ return_in_uV=False,
20
+ ):
21
+ assert times_samples.ndim == 1
22
+ assert times_samples.size > 0
23
+ assert times_samples.dtype.kind == "i"
24
+ assert times_samples.max() <= recording.get_num_samples() - (
25
+ spike_length_samples - trough_offset_samples
26
+ )
27
+ n_channels = recording.get_num_channels()
28
+ n_spikes = times_samples.size
29
+ read_times = times_samples - trough_offset_samples
30
+
31
+ if not return_in_uV and recording.binary_compatible_with(
32
+ file_offset=0, time_axis=0, file_paths_length=1
33
+ ):
34
+ # fast path. this is like 2x as fast as the read_traces for loop
35
+ # below, but requires a recording on disk in a nice format
36
+ binary_path = recording.get_binary_description()["file_paths"][0]
37
+ if channel_subset is None:
38
+ return _read_full_waveforms_binary(
39
+ binary_path,
40
+ read_times,
41
+ n_channels=n_channels,
42
+ dtype=recording.dtype,
43
+ spike_length_samples=spike_length_samples,
44
+ )
45
+ else:
46
+ return _read_subset_waveforms_binary(
47
+ binary_path,
48
+ times_samples,
49
+ n_channels,
50
+ dtype=recording.dtype,
51
+ load_channels=channel_subset,
52
+ trough_offset_samples=trough_offset_samples,
53
+ spike_length_samples=spike_length_samples,
54
+ )
55
+
56
+ waveforms = np.empty(
57
+ (n_spikes, spike_length_samples, n_channels), dtype=recording.dtype
58
+ )
59
+ for i, t in enumerate(read_times):
60
+ waveforms[i] = recording.get_traces(
61
+ 0,
62
+ start_frame=t,
63
+ end_frame=t + spike_length_samples,
64
+ return_in_uV=return_in_uV,
65
+ )
66
+ if channel_subset is not None:
67
+ waveforms = waveforms[:, :, channel_subset]
68
+
69
+ return waveforms
70
+
71
+
72
+ def read_waveforms_channel_index(
73
+ recording,
74
+ times_samples,
75
+ channel_index,
76
+ main_channels,
77
+ trough_offset_samples=42,
78
+ spike_length_samples=121,
79
+ fill_value=np.nan,
80
+ ):
81
+ assert times_samples.ndim == 1
82
+ assert times_samples.size > 0
83
+ assert times_samples.dtype.kind == "i"
84
+ assert times_samples.min() >= trough_offset_samples
85
+ assert times_samples.max() <= recording.get_num_samples() - (
86
+ spike_length_samples - trough_offset_samples
87
+ )
88
+ n_channels = recording.get_num_channels()
89
+
90
+ if recording.binary_compatible_with(
91
+ file_offset=0, time_axis=0, file_paths_length=1
92
+ ):
93
+ # fast path. this is like 2x as fast as the read_traces for loop
94
+ # below, but requires a recording on disk in a nice format
95
+ binary_path = recording.get_binary_description()["file_paths"][0]
96
+ return _read_waveforms_binary_channel_index(
97
+ binary_path,
98
+ times_samples,
99
+ channel_index=channel_index,
100
+ main_channels=main_channels,
101
+ n_channels=n_channels,
102
+ dtype=recording.dtype,
103
+ trough_offset_samples=trough_offset_samples,
104
+ spike_length_samples=spike_length_samples,
105
+ fill_value=fill_value,
106
+ )
107
+
108
+ n_spikes = times_samples.size
109
+ waveforms = np.full(
110
+ (n_spikes, spike_length_samples, channel_index.shape[1]),
111
+ fill_value,
112
+ dtype=recording.dtype,
113
+ )
114
+ read_times = times_samples - trough_offset_samples
115
+
116
+ for i, t in enumerate(read_times):
117
+ chans = channel_index[main_channels[i]]
118
+ good = chans < n_channels
119
+ load_channel_ids = recording.channel_ids[chans[good]]
120
+ waveforms[i, :, good] = recording.get_traces(
121
+ 0,
122
+ start_frame=t,
123
+ end_frame=t + spike_length_samples,
124
+ channel_ids=load_channel_ids,
125
+ ).T
126
+
127
+ return waveforms
128
+
129
+
130
+ def read_single_channel_waveforms(
131
+ recording,
132
+ times_samples,
133
+ channels,
134
+ trough_offset_samples=42,
135
+ spike_length_samples=121,
136
+ fill_value=np.nan,
137
+ ):
138
+ single_channel_index = np.arange(recording.get_num_channels())[:, None]
139
+ return read_waveforms_channel_index(
140
+ recording,
141
+ times_samples,
142
+ channel_index=single_channel_index,
143
+ main_channels=channels,
144
+ trough_offset_samples=trough_offset_samples,
145
+ spike_length_samples=spike_length_samples,
146
+ fill_value=fill_value,
147
+ )
148
+
149
+
150
+ def _read_full_waveforms_binary(
151
+ binary_path,
152
+ read_times_samples,
153
+ n_channels,
154
+ dtype,
155
+ spike_length_samples=121,
156
+ ):
157
+ n_spikes = read_times_samples.size
158
+ waveforms = np.empty((n_spikes, spike_length_samples, n_channels), dtype=dtype)
159
+ offsets = read_times_samples * np.dtype(dtype).itemsize * n_channels
160
+ with open(binary_path, "rb") as binary:
161
+ for i, offset in enumerate(offsets):
162
+ binary.seek(offset, SEEK_SET)
163
+ waveforms[i] = np.fromfile(
164
+ binary,
165
+ dtype=dtype,
166
+ count=spike_length_samples * n_channels,
167
+ ).reshape(spike_length_samples, n_channels)
168
+ return waveforms
169
+
170
+
171
+ def _read_subset_waveforms_binary(
172
+ binary_path,
173
+ times_samples,
174
+ n_channels,
175
+ dtype,
176
+ load_channels,
177
+ trough_offset_samples=42,
178
+ spike_length_samples=121,
179
+ ):
180
+ n_spikes = times_samples.size
181
+ waveforms = np.empty(
182
+ (n_spikes, spike_length_samples, load_channels.size), dtype=dtype
183
+ )
184
+ load_times = times_samples - trough_offset_samples
185
+ offsets = load_times * np.dtype(dtype).itemsize * n_channels
186
+ with open(binary_path, "rb") as binary:
187
+ for i, offset in enumerate(offsets):
188
+ binary.seek(offset, SEEK_SET)
189
+ waveforms[i] = np.fromfile(
190
+ binary,
191
+ dtype=dtype,
192
+ count=spike_length_samples * n_channels,
193
+ ).reshape(spike_length_samples, n_channels)[:, load_channels]
194
+ return waveforms
195
+
196
+
197
+ def _read_waveforms_binary_channel_index(
198
+ binary_path,
199
+ times_samples,
200
+ n_channels,
201
+ dtype,
202
+ channel_index,
203
+ main_channels,
204
+ trough_offset_samples=42,
205
+ spike_length_samples=121,
206
+ fill_value=np.nan,
207
+ ):
208
+ n_spikes = times_samples.size
209
+ waveforms = np.full(
210
+ (n_spikes, spike_length_samples, channel_index.shape[1]),
211
+ fill_value,
212
+ dtype=dtype,
213
+ )
214
+ load_times = times_samples - trough_offset_samples
215
+ offsets = load_times * np.dtype(dtype).itemsize * n_channels
216
+ with open(binary_path, "rb") as binary:
217
+ for i, offset in enumerate(offsets):
218
+ binary.seek(offset, SEEK_SET)
219
+ wf = np.fromfile(
220
+ binary,
221
+ dtype=dtype,
222
+ count=spike_length_samples * n_channels,
223
+ ).reshape(spike_length_samples, n_channels)
224
+ chans = channel_index[main_channels[i]]
225
+ good = chans < n_channels
226
+ waveforms[i, :, good] = wf.T[chans[good]]
227
+ return waveforms