DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,219 @@
1
+ import numpy as np
2
+ from tqdm.auto import trange
3
+
4
+
5
+ def forward_backward(
6
+ chunk_time_ranges_s,
7
+ chunk_sortings,
8
+ log_c=5.0,
9
+ feature_scales=(1, 1, 50),
10
+ adaptive_feature_scales=False,
11
+ motion_est=None,
12
+ verbose=True,
13
+ min_cluster_size=25,
14
+ ):
15
+ """
16
+ Ensemble over HDBSCAN clustering
17
+ triaging/subsampling/copying/splitting big clusters not implemented since we don't use it (so far)
18
+ """
19
+ if len(chunk_sortings) == 1:
20
+ return chunk_sortings[0].labels
21
+
22
+ times_seconds = chunk_sortings[0].times_seconds
23
+
24
+ min_time_s = chunk_time_ranges_s[0][0]
25
+ idx_all_chunks = [
26
+ get_indices_in_chunk(times_seconds, chunk_range)
27
+ for chunk_range in chunk_time_ranges_s
28
+ ]
29
+
30
+ # put all labels into one array
31
+ # TODO: this does not allow for overlapping chunks.
32
+ labels_all = np.full_like(times_seconds, -1)
33
+ for ix, sorting in zip(idx_all_chunks, chunk_sortings):
34
+ if len(ix): # type: ignore
35
+ assert labels_all[ix].max() < 0 # assert non-overlapping
36
+ labels_all[ix] = sorting.labels[ix]
37
+
38
+ # load features that we will need
39
+ # needs to be all features here
40
+ amps = chunk_sortings[0].denoised_ptp_amplitudes
41
+ xyza = chunk_sortings[0].point_source_localizations
42
+
43
+ x = xyza[:, 0]
44
+ z_reg = xyza[:, 2]
45
+
46
+ if adaptive_feature_scales:
47
+ feature_scales = (
48
+ 1,
49
+ 1,
50
+ np.median(np.abs(x - np.median(x)))
51
+ / np.median(np.abs(np.log(log_c + amps) - np.median(np.log(log_c + amps)))),
52
+ )
53
+
54
+ if motion_est is not None:
55
+ z_reg = motion_est.correct_s(times_seconds, z_reg)
56
+
57
+ if verbose is True:
58
+ tbar = trange(len(chunk_sortings) - 1, desc="Ensembling chunks")
59
+ else:
60
+ tbar = range(len(chunk_sortings) - 1)
61
+ for k in tbar:
62
+ idx_1 = np.flatnonzero(
63
+ np.logical_and(
64
+ times_seconds >= min_time_s,
65
+ times_seconds < chunk_time_ranges_s[k][1],
66
+ )
67
+ )
68
+ idx_2 = idx_all_chunks[k + 1]
69
+ x_1 = feature_scales[0] * x[idx_1]
70
+ x_2 = feature_scales[0] * x[idx_2]
71
+ z_1 = feature_scales[1] * z_reg[idx_1]
72
+ z_2 = feature_scales[1] * z_reg[idx_2]
73
+ amps_1 = feature_scales[2] * np.log(log_c + amps[idx_1])
74
+ amps_2 = feature_scales[2] * np.log(log_c + amps[idx_2])
75
+ labels_1 = labels_all[idx_1].copy().astype("int")
76
+ labels_2 = chunk_sortings[k + 1].labels[idx_2]
77
+
78
+ units_1 = np.unique(labels_1)
79
+ units_1 = units_1[units_1 > -1]
80
+ units_2 = np.unique(labels_2)
81
+ units_2 = units_2[units_2 > -1]
82
+
83
+ features1 = np.c_[x_1, z_1, amps_1]
84
+ features2 = np.c_[x_2, z_2, amps_2]
85
+
86
+ if len(units_2) and len(units_1):
87
+ unit_label_shift = int(labels_1.max() + 1)
88
+ labels_2[labels_2 > -1] += unit_label_shift
89
+ units_2 += unit_label_shift
90
+
91
+ # FORWARD PASS
92
+ dist_matrix = np.zeros((units_1.shape[0], units_2.shape[0]))
93
+ for i, unit_1 in enumerate(units_1):
94
+ for j, unit_2 in enumerate(units_2):
95
+ idxunit1 = np.flatnonzero(labels_1 == unit_1)[-min_cluster_size:]
96
+ idxunit2 = np.flatnonzero(labels_2 == unit_2)[:min_cluster_size]
97
+ feat_1 = np.median(features1[idxunit1], axis=0)
98
+ feat_2 = np.median(features2[idxunit2], axis=0)
99
+ dist_matrix[i, j] = ((feat_1 - feat_2) ** 2).sum()
100
+
101
+ # find for chunk 2 units the closest units in chunk 1 and split chunk 1 units
102
+ dist_forward = dist_matrix.argmin(0)
103
+ units_, counts_ = np.unique(dist_forward, return_counts=True)
104
+
105
+ for unit_to_split in units_[counts_ > 1]:
106
+ units_to_match_to = (
107
+ np.flatnonzero(dist_forward == unit_to_split) + unit_label_shift
108
+ )
109
+
110
+ idxunit2 = np.flatnonzero(labels_2 == units_to_match_to[0])
111
+ features_to_match_to = np.median(features2[idxunit2], axis=0)
112
+
113
+ for u in units_to_match_to[1:]:
114
+ idxunit2 = np.flatnonzero(labels_2 == u)
115
+ features_to_match_to = np.c_[
116
+ features_to_match_to, np.median(features2[idxunit2], axis=0)
117
+ ]
118
+
119
+ spikes_to_update = np.flatnonzero(labels_1 == unit_to_split)
120
+ feat_s = features1[spikes_to_update]
121
+ labels_1[spikes_to_update] = units_to_match_to[
122
+ ((features_to_match_to.T[:, None] - feat_s[None]) ** 2)
123
+ .sum(2)
124
+ .argmin(0)
125
+ ]
126
+
127
+ # Relabel labels_1 and labels_2
128
+ for unit_to_relabel in units_:
129
+ if counts_[np.flatnonzero(units_ == unit_to_relabel)][0] == 1:
130
+ idx_to_relabel = np.flatnonzero(labels_1 == unit_to_relabel)
131
+ labels_1[idx_to_relabel] = units_2[dist_forward == unit_to_relabel]
132
+
133
+ # BACKWARD PASS
134
+ units_not_matched = np.unique(labels_1)
135
+ units_not_matched = units_not_matched[units_not_matched > -1]
136
+ units_not_matched = units_not_matched[units_not_matched < unit_label_shift]
137
+
138
+ if len(units_not_matched):
139
+ all_units_to_match_to = (
140
+ dist_matrix[units_not_matched].argmin(1) + unit_label_shift
141
+ )
142
+ for unit_to_split in np.unique(all_units_to_match_to):
143
+ units_to_match_to = np.concatenate(
144
+ (
145
+ units_not_matched[all_units_to_match_to == unit_to_split],
146
+ [unit_to_split],
147
+ )
148
+ )
149
+
150
+ features_to_match_to = np.median(
151
+ features1[labels_1 == units_to_match_to[0]], axis=0
152
+ )
153
+ for u in units_to_match_to[1:]:
154
+ features_to_match_to = np.c_[
155
+ features_to_match_to,
156
+ np.median(features1[labels_1 == u], axis=0),
157
+ ]
158
+
159
+ spikes_to_update = np.flatnonzero(labels_2 == unit_to_split)
160
+ features2[spikes_to_update]
161
+ labels_2[spikes_to_update] = units_to_match_to[
162
+ (
163
+ (
164
+ features_to_match_to.T[:, None]
165
+ - features2[spikes_to_update][None]
166
+ )
167
+ ** 2
168
+ )
169
+ .sum(2)
170
+ .argmin(0)
171
+ ]
172
+
173
+ # Do we need to "regularize" and make sure the distance intra units after merging is smaller than the distance inter units before merging
174
+ # all_labels_1 = np.unique(labels_1)
175
+ # all_labels_1 = all_labels_1[all_labels_1 > -1]
176
+
177
+ # features_all_1 = np.c_[
178
+ # np.median(x_1[labels_1 == all_labels_1[0]]),
179
+ # np.median(z_1[labels_1 == all_labels_1[0]]),
180
+ # np.median(amps_1[labels_1 == all_labels_1[0]]),
181
+ # ]
182
+ # for u in all_labels_1[1:]:
183
+ # features_all_1 = np.concatenate(
184
+ # (
185
+ # features_all_1,
186
+ # np.c_[
187
+ # np.median(x_1[labels_1 == u]),
188
+ # np.median(z_1[labels_1 == u]),
189
+ # np.median(amps_1[labels_1 == u]),
190
+ # ],
191
+ # )
192
+ # )
193
+
194
+ # distance_inter = (
195
+ # (features_all_1[:, :, None] - features_all_1.T[None]) ** 2
196
+ # ).sum(1)
197
+
198
+ labels_12 = np.concatenate((labels_1, labels_2))
199
+ _, labels_12[labels_12 > -1] = np.unique(
200
+ labels_12[labels_12 > -1], return_inverse=True
201
+ ) # Make contiguous
202
+ idx_all = np.flatnonzero(
203
+ times_seconds < min_time_s + chunk_time_ranges_s[k + 1][1]
204
+ )
205
+ labels_all = -1 * np.ones(
206
+ times_seconds.shape[0]
207
+ ) # discard all spikes at the end for now
208
+ labels_all[idx_all] = labels_12.astype("int")
209
+
210
+ return labels_all
211
+
212
+
213
+ def get_indices_in_chunk(times_s, chunk_time_range_s):
214
+ if chunk_time_range_s is None:
215
+ return slice(None)
216
+
217
+ return np.flatnonzero(
218
+ (times_s >= chunk_time_range_s[0]) & (times_s < chunk_time_range_s[1])
219
+ )