DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,359 @@
1
+ from pathlib import Path
2
+ from typing import Optional
3
+
4
+ import h5py
5
+ import numpy as np
6
+ import torch
7
+ import torch.nn.functional as F
8
+
9
+ from ...templates.template_util import CompressedUpsampledTemplates, LowRankTemplates
10
+ from ...templates.templates import TemplateData
11
+ from ...util.py_util import resolve_path
12
+ from ...util import job_util
13
+ from .pairwise_util import compressed_convolve_to_h5
14
+ from .matching_base import PconvBase
15
+
16
+
17
+ class CompressedPairwiseConv(PconvBase):
18
+ """A database of channel-summed cross-correlations between template pairs
19
+
20
+ There are too many templates to store all of these, especially after
21
+ superres binning, temporal upsampling, and pitch shifting. We compress
22
+ this as much as possible, first by deduplication (many convolutions of
23
+ templates at different shifts are identical), next by not wasting space
24
+ (no need to compute as many upsampled copies of small templates), and
25
+ finally by approximation (for pairs of far-away units, correlations of
26
+ superres templates are very close to correlations of the non-superres
27
+ template).
28
+
29
+ This database holds some indexing structures that help us store these
30
+ correlations sparsely. .query() grabs the actual correlations for the
31
+ user.
32
+ """
33
+
34
+ # shifts_b
35
+ # shape: (n_shifts,)
36
+ # shift_ix -> shift (pitch shift, an integer)
37
+
38
+ # shifted_template_index_a
39
+ # shape: (n_templates_a, n_shifts_a)
40
+ # (template_ix, shift_ix) -> shifted_template_ix
41
+ # shifted_template_ix can be either invalid (this template does not occur
42
+ # at this shift), or it can range from 0, ..., n_shifted_templates_a-1
43
+
44
+ # upsampled_shifted_template_index_b
45
+ # shape: (n_templates_b, n_shifts_b, upsampling_factor)
46
+ # (template_ix, shift_ix, upsampling_ix) -> upsampled_shifted_template_ix
47
+
48
+ # pconv_index
49
+ # shape: (n_shifted_templates_a, n_upsampled_shifted_templates_b)
50
+ # (shifted_template_ix, upsampled_shifted_template_ix) -> pconv_ix
51
+
52
+ # pconv
53
+ # shape: (n_pconvs, 2 * spike_length_samples - 1)
54
+ # pconv_ix -> a cross-correlation array
55
+ # the 0 index is special: pconv[0] === 0.
56
+
57
+ def __init__(
58
+ self,
59
+ pconv: torch.Tensor,
60
+ pconv_index: torch.Tensor,
61
+ shifted_template_index_a: torch.Tensor,
62
+ upsampled_shifted_template_index_b: torch.Tensor,
63
+ shifts_a: torch.Tensor | None = None,
64
+ shifts_b: torch.Tensor | None = None,
65
+ on_device: bool = False,
66
+ ):
67
+ super().__init__()
68
+ self.on_device = on_device
69
+ self.register_buffer_or_none("pconv_index", pconv_index)
70
+ self.register_buffer_or_none("pconv", pconv, on_device=on_device)
71
+ self.register_buffer_or_none(
72
+ "shifted_template_index_a", shifted_template_index_a
73
+ )
74
+ self.register_buffer_or_none(
75
+ "upsampled_shifted_template_index_b",
76
+ upsampled_shifted_template_index_b,
77
+ )
78
+ self.register_buffer_or_none("shifts_a", shifts_a)
79
+ self.register_buffer_or_none("shifts_b", shifts_b)
80
+
81
+ # helper bufs
82
+ na = len(self.b.shifted_template_index_a)
83
+ self.register_buffer_or_none("all_inds_a", torch.arange(na))
84
+
85
+ self.not_shifting = self.b.shifts_a is None
86
+ assert self.not_shifting == (self.b.shifts_b is None)
87
+ self.shifting = not self.not_shifting
88
+
89
+ if self.shifting:
90
+ shoffset_a, shindex_a = _get_shift_indexer(shifts_a)
91
+ shoffset_a = torch.tensor(shoffset_a)
92
+ shoffset_b, shindex_b = _get_shift_indexer(shifts_b)
93
+ shoffset_b = torch.tensor(shoffset_b)
94
+ else:
95
+ shoffset_a = shindex_a = None
96
+ shoffset_b = shindex_b = None
97
+ self.register_buffer_or_none("shoffset_a", shoffset_a)
98
+ self.register_buffer_or_none("shindex_a", shindex_a)
99
+ self.register_buffer_or_none("shoffset_b", shoffset_b)
100
+ self.register_buffer_or_none("shindex_b", shindex_b)
101
+
102
+ @classmethod
103
+ def from_h5(cls, hdf5_filename, on_device=False):
104
+ with h5py.File(hdf5_filename, "r", locking=False) as h5:
105
+ pconv = torch.from_numpy(h5["pconv"][:]) # type: ignore
106
+ pconv_index = torch.from_numpy(h5["pconv_index"][:]) # type: ignore
107
+ shifted_template_index_a = torch.from_numpy(
108
+ h5["shifted_template_index_a"][:] # type: ignore
109
+ )
110
+ upsampled_shifted_template_index_b = torch.from_numpy(
111
+ h5["upsampled_shifted_template_index_b"][:] # type: ignore
112
+ )
113
+ if "shifts_a" in h5:
114
+ assert "shifts_b" in h5
115
+ shifts_a = torch.from_numpy(h5["shifts_a"][:]) # type: ignore
116
+ shifts_b = torch.from_numpy(h5["shifts_b"][:]) # type: ignore
117
+ else:
118
+ shifts_a = shifts_b = None
119
+ return cls(
120
+ pconv=pconv,
121
+ pconv_index=pconv_index,
122
+ shifted_template_index_a=shifted_template_index_a,
123
+ upsampled_shifted_template_index_b=upsampled_shifted_template_index_b,
124
+ shifts_a=shifts_a,
125
+ shifts_b=shifts_b,
126
+ on_device=on_device,
127
+ )
128
+
129
+ @classmethod
130
+ def from_template_data(
131
+ cls,
132
+ hdf5_filename: str | Path,
133
+ template_data: TemplateData,
134
+ low_rank_templates: LowRankTemplates,
135
+ compressed_upsampled_temporal: CompressedUpsampledTemplates,
136
+ template_data_b: Optional[TemplateData] = None,
137
+ low_rank_templates_b: Optional[LowRankTemplates] = None,
138
+ chunk_time_centers_s: Optional[np.ndarray] = None,
139
+ motion_est=None,
140
+ geom: Optional[np.ndarray] = None,
141
+ conv_batch_size=1024,
142
+ units_batch_size=8,
143
+ overwrite=False,
144
+ computation_cfg=None,
145
+ show_progress=True,
146
+ ):
147
+ if computation_cfg is None:
148
+ computation_cfg = job_util.get_global_computation_config()
149
+
150
+ hdf5_filename = resolve_path(hdf5_filename)
151
+ hdf5_filename.parent.mkdir(exist_ok=True)
152
+
153
+ # TODO: rewrite.
154
+ compressed_convolve_to_h5(
155
+ hdf5_filename,
156
+ template_data=template_data,
157
+ low_rank_templates=low_rank_templates,
158
+ compressed_upsampled_temporal=compressed_upsampled_temporal,
159
+ template_data_b=template_data_b,
160
+ low_rank_templates_b=low_rank_templates_b,
161
+ chunk_time_centers_s=chunk_time_centers_s,
162
+ motion_est=motion_est,
163
+ geom=geom,
164
+ conv_batch_size=conv_batch_size,
165
+ units_batch_size=units_batch_size,
166
+ overwrite=overwrite,
167
+ device=computation_cfg.actual_device(),
168
+ n_jobs=computation_cfg.actual_n_jobs(),
169
+ show_progress=show_progress,
170
+ )
171
+ return cls.from_h5(hdf5_filename)
172
+
173
+ def get_shift_ix_a(self, shifts_a):
174
+ """Map shift (an integer, signed) to a shift index
175
+
176
+ A shift index can be used to index into axis=1 of shifted_template_index_a,
177
+ or self.shifts_a for that matter.
178
+ It's an int in [0, n_shifts_a).
179
+ It's equal to np.searchsorted(self.shifts_a, shifts_a).
180
+ The thing is, searchsorted is slow, and we can pre-bake a lookup table.
181
+ _get_shift_indexer does the baking for us below.
182
+ """
183
+ shifts_a = torch.atleast_1d(shifts_a)
184
+ return self.b.shindex_a[shifts_a.int() + self.b.shoffset_a]
185
+
186
+ def get_shift_ix_b(self, shifts_b):
187
+ shifts_b = torch.atleast_1d(shifts_b)
188
+ return self.b.shindex_b[shifts_b.int() + self.b.shoffset_b]
189
+
190
+ def query(
191
+ self,
192
+ template_indices_a,
193
+ template_indices_b,
194
+ upsampling_indices_b=None,
195
+ shifts_a=None,
196
+ shifts_b=None,
197
+ scalings_b=None,
198
+ ):
199
+ if template_indices_a is None:
200
+ template_indices_a = self.b.all_inds_a
201
+ dev = self.b.all_inds_a.device
202
+ template_indices_a = torch.atleast_1d(torch.asarray(template_indices_a, device=dev))
203
+ template_indices_b = torch.atleast_1d(torch.asarray(template_indices_b, device=dev))
204
+
205
+ # handle no shifting
206
+ no_shifting = (shifts_a is None) or (shifts_b is None)
207
+ shifted_template_index = self.b.shifted_template_index_a
208
+ upsampled_shifted_template_index = self.b.upsampled_shifted_template_index_b
209
+ if no_shifting:
210
+ assert shifts_a is None and shifts_b is None
211
+ assert self.b.shifts_a.shape == (1,)
212
+ assert self.b.shifts_b.shape == (1,)
213
+ a_ix = (template_indices_a,)
214
+ b_ix = (template_indices_b,)
215
+ shifted_template_index = shifted_template_index[:, 0]
216
+ upsampled_shifted_template_index = upsampled_shifted_template_index[:, 0]
217
+ else:
218
+ shift_indices_a = self.get_shift_ix_a(shifts_a)
219
+ shift_indices_b = self.get_shift_ix_a(shifts_b)
220
+ a_ix = (template_indices_a, shift_indices_a)
221
+ b_ix = (template_indices_b, shift_indices_b)
222
+
223
+ # handle no upsampling
224
+ no_upsampling = upsampling_indices_b is None
225
+ if no_upsampling:
226
+ assert self.b.upsampled_shifted_template_index_b.shape[2] == 1
227
+ upsampled_shifted_template_index = upsampled_shifted_template_index[..., 0]
228
+ else:
229
+ b_ix = b_ix + (torch.atleast_1d(torch.asarray(upsampling_indices_b, device=dev)),)
230
+
231
+ # get shifted template indices for A
232
+ shifted_temp_ix_a = shifted_template_index[a_ix]
233
+
234
+ # upsampled shifted template indices for B
235
+ up_shifted_temp_ix_b = upsampled_shifted_template_index[b_ix]
236
+
237
+ pconv_indices = self.b.pconv_index[
238
+ shifted_temp_ix_a[:, None], up_shifted_temp_ix_b[None, :]
239
+ ].view(-1)
240
+ (which,) = pconv_indices.nonzero(as_tuple=True)
241
+ which_a = which // up_shifted_temp_ix_b.shape[0]
242
+ which_b = which % up_shifted_temp_ix_b.shape[0]
243
+ template_indices_a = template_indices_a[which_a]
244
+ pconv_indices = pconv_indices[which]
245
+
246
+ if self.on_device:
247
+ pconvs = self.b.pconv[pconv_indices]
248
+ else:
249
+ pconvs = self.b.pconv[pconv_indices.cpu()]
250
+ if scalings_b is not None:
251
+ pconvs = pconvs.to(scalings_b.device)
252
+ pconvs.mul_(scalings_b[which_b].unsqueeze(1))
253
+
254
+ return template_indices_a, pconvs, which_b
255
+
256
+
257
+ class SeparablePairwiseConv(PconvBase):
258
+ def __init__(self, spatial_footprints, temporal_shapes):
259
+ """Footprint-major rank 1 template convolution database
260
+
261
+ Let Nf = len(spatial_footprints), Ns = len(temporal_shapes). Then
262
+ indexing is footprint-major, so that
263
+
264
+ template[i] = spatial_footprints[i // Ns] * temporal_shapes[i - Ns * (i // Ns)]
265
+
266
+ Let f(i) = i // Ns and s(i) = i - Ns * (i // Ns). Then the channel-summed
267
+ convolution of templates i and j is given by
268
+
269
+ conv(t; i, j) = (
270
+ <spatial_footprints[f(i)], spatial_footprints[f(j)]>
271
+ * conv1d(temporal_shapes[s(i)], temporal_shapes[s(j)])[t]
272
+ )
273
+
274
+ Note: need to be consistent with interpretation of the sign of the time lag
275
+ between here and CompressedPairwiseConv.
276
+ """
277
+ super().__init__()
278
+ self.register_buffer("spatial_footprints", torch.asarray(spatial_footprints))
279
+ self.register_buffer("temporal_shapes", torch.asarray(temporal_shapes))
280
+ self.Nf = len(spatial_footprints)
281
+ self.Ns, self.nt = temporal_shapes.shape
282
+
283
+ # convolve all pairs of temporal shapes
284
+ # i is data, j is filter
285
+ nt = temporal_shapes.shape[1]
286
+ inp = self.b.temporal_shapes[:, None, :]
287
+ fil = self.b.temporal_shapes[:, None, :]
288
+ # Ns, Ns, 2*nt - 1
289
+ self.register_buffer("tconv", F.conv1d(inp, fil, padding=nt - 1))
290
+
291
+ # spatial component
292
+ sdot = self.b.spatial_footprints @ self.b.spatial_footprints.T
293
+ self.register_buffer("sdot", sdot)
294
+ self.register_buffer("overlap", self.b.sdot > 0)
295
+ self.tia = torch.arange(self.Ns * self.Nf)
296
+
297
+ @property
298
+ def device(self):
299
+ return self.tconv.device
300
+
301
+ def query(
302
+ self,
303
+ template_indices_a,
304
+ template_indices_b,
305
+ upsampling_indices_b=None,
306
+ shifts_a=None,
307
+ shifts_b=None,
308
+ scalings_b=None,
309
+ ):
310
+ assert shifts_a is shifts_b is None
311
+ assert upsampling_indices_b is None or (upsampling_indices_b == 0).all()
312
+ if template_indices_a is None:
313
+ template_indices_a = self.tia.to(template_indices_b)
314
+
315
+ f_i = template_indices_a // self.Ns
316
+ f_j = template_indices_b // self.Ns
317
+ keep_i, keep_j = self.b.overlap[f_i[:, None], f_j[None, :]].nonzero(
318
+ as_tuple=True
319
+ )
320
+
321
+ template_indices_a = template_indices_a[keep_i]
322
+ template_indices_b = template_indices_b[keep_j]
323
+
324
+ f_i = template_indices_a // self.Ns
325
+ f_j = template_indices_b // self.Ns
326
+ s_i = template_indices_a - self.Ns * f_i
327
+ s_j = template_indices_b - self.Ns * f_j
328
+
329
+ sdot = self.b.sdot[f_i, f_j]
330
+ assert sdot.ndim == 1
331
+ if scalings_b is not None:
332
+ sdot = sdot * scalings_b[keep_j]
333
+ tconv = self.b.tconv[s_i, s_j]
334
+ assert tconv.shape == (len(sdot), 2 * self.nt - 1)
335
+ pconvs = sdot.unsqueeze(1) * tconv
336
+
337
+ return template_indices_a, pconvs, keep_j
338
+
339
+
340
+ def _get_shift_indexer(shifts):
341
+ assert torch.equal(shifts, torch.sort(shifts).values)
342
+ # smallest shift (say, -5) becomes 5
343
+ shift_offset = -int(shifts[0])
344
+ offset_shift_to_ix = []
345
+
346
+ for j, shift in enumerate(shifts):
347
+ ix = shift + shift_offset
348
+ assert len(offset_shift_to_ix) <= ix
349
+
350
+ # fill indices corresponding to missing shifts with an out-of-bounds
351
+ # index to cause a panic if someone tries to load up a shift which DNE
352
+ while len(offset_shift_to_ix) < ix:
353
+ offset_shift_to_ix.append(len(shifts))
354
+
355
+ # real shifts get good index
356
+ offset_shift_to_ix.append(j)
357
+
358
+ offset_shift_to_ix = torch.tensor(offset_shift_to_ix, device=shifts.device)
359
+ return shift_offset, offset_shift_to_ix