DARTsort 0.5.9__tar.gz → 0.5.11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. {dartsort-0.5.9 → dartsort-0.5.11}/PKG-INFO +2 -2
  2. {dartsort-0.5.9 → dartsort-0.5.11}/README.md +1 -1
  3. {dartsort-0.5.9 → dartsort-0.5.11}/docs/index.md +1 -1
  4. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/agglomerate.py +295 -31
  5. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/cluster_util.py +4 -3
  6. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/clustering.py +78 -1
  7. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/clustering_features.py +37 -12
  8. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/kmeans.py +296 -5
  9. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/mixture.py +274 -65
  10. dartsort-0.5.11/src/dartsort/clustering/refine_util.py +775 -0
  11. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/config.py +29 -8
  12. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/analysis.py +48 -7
  13. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/comparison.py +43 -5
  14. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/main.py +18 -11
  15. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/grab.py +89 -16
  16. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching.py +24 -6
  17. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/compressed_upsampled.py +12 -1
  18. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/drifty.py +79 -37
  19. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/matching_base.py +7 -2
  20. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/peel_base.py +55 -16
  21. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/peel_lib.py +69 -29
  22. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/reduction_template.py +14 -9
  23. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/subtract.py +59 -10
  24. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/get_templates.py +2 -2
  25. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/postprocess_util.py +88 -32
  26. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/template_util.py +7 -3
  27. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/templates.py +10 -4
  28. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/templib.py +5 -0
  29. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/_multichan_denoiser_kit.py +12 -2
  30. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/amortized_localization.py +5 -5
  31. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/decollider.py +62 -12
  32. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/mixture_classifier.py +4 -1
  33. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/pipeline.py +7 -1
  34. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/temporal_pca.py +39 -14
  35. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/transform_base.py +22 -3
  36. dartsort-0.5.11/src/dartsort/transform/whiten.py +102 -0
  37. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/cli_util.py +0 -2
  38. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/data_util.py +278 -12
  39. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/internal_config.py +114 -35
  40. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/main_util.py +7 -4
  41. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/noise_util.py +180 -27
  42. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/spiketorch.py +24 -5
  43. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/testing_util/matching_debug_util.py +40 -6
  44. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/analysis_plots.py +166 -5
  45. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/gt.py +39 -3
  46. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/mixture.py +6 -6
  47. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/sorting.py +60 -8
  48. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/unit.py +194 -84
  49. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/unit_comparison.py +12 -6
  50. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/vismain.py +9 -4
  51. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/waveforms.py +1 -1
  52. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/PKG-INFO +2 -2
  53. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_clustering.py +5 -2
  54. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_config.py +2 -1
  55. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_dartsort.py +1 -1
  56. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_kmeans.py +7 -4
  57. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_matching.py +2 -0
  58. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_subtract.py +8 -9
  59. dartsort-0.5.9/src/dartsort/clustering/refine_util.py +0 -181
  60. dartsort-0.5.9/src/dartsort/transform/whiten.py +0 -34
  61. {dartsort-0.5.9 → dartsort-0.5.11}/.github/workflows/ci.yml +0 -0
  62. {dartsort-0.5.9 → dartsort-0.5.11}/.github/workflows/ci_windows.yml +0 -0
  63. {dartsort-0.5.9 → dartsort-0.5.11}/.github/workflows/deploy.yml +0 -0
  64. {dartsort-0.5.9 → dartsort-0.5.11}/.github/workflows/docs.yml +0 -0
  65. {dartsort-0.5.9 → dartsort-0.5.11}/.gitignore +0 -0
  66. {dartsort-0.5.9 → dartsort-0.5.11}/docs/main_api.md +0 -0
  67. {dartsort-0.5.9 → dartsort-0.5.11}/docs/toolkit_api.md +0 -0
  68. {dartsort-0.5.9 → dartsort-0.5.11}/environment.yml +0 -0
  69. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/.keep +0 -0
  70. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/basic_integration_test.py +0 -0
  71. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/ci diag.py +0 -0
  72. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/dartsort_dredge_demo.ipynb +0 -0
  73. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/dartsort_dredge_demo.py +0 -0
  74. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/data test.py +0 -0
  75. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/dead reckoning.py +0 -0
  76. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/denoise np1.py +0 -0
  77. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/denoise np2.py +0 -0
  78. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/exploratory.py +0 -0
  79. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/geom debug.py +0 -0
  80. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/hdbscan_clustering_5min.py +0 -0
  81. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/hybrid 10.py +0 -0
  82. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/hybrid cluster.py +0 -0
  83. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/hybrid_11.py +0 -0
  84. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/initial training.py +0 -0
  85. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/make hybrid.py +0 -0
  86. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/mgh.py +0 -0
  87. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/np1 feats.py +0 -0
  88. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/np2 processing version c.py +0 -0
  89. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/paperfigs.py +0 -0
  90. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/pdists.py +0 -0
  91. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/postcheck.py +0 -0
  92. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/ptp vae.py +0 -0
  93. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/re clust.py +0 -0
  94. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/reg fig loop.py +0 -0
  95. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/reg_sim_exp.py +0 -0
  96. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/relocating.py +0 -0
  97. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/relocating_all.py +0 -0
  98. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/restats +0 -0
  99. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/sortStinger1.py +0 -0
  100. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/sortStinger3.py +0 -0
  101. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/sortStinger4.py +0 -0
  102. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/sortStinger5.py +0 -0
  103. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/sortStinger6.py +0 -0
  104. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/spike farm.py +0 -0
  105. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/ss_ensemble_sim.py +0 -0
  106. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/ss_ensemble_sim_drift.py +0 -0
  107. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/ss_sim.py +0 -0
  108. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/stan diagnostics.py +0 -0
  109. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/storytelling c.py +0 -0
  110. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/storytelling.py +0 -0
  111. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
  112. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/debug_matching.py +0 -0
  113. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_agreement.py +0 -0
  114. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_amplitude_scaling.py +0 -0
  115. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_batch_optim.py +0 -0
  116. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_ibme.py +0 -0
  117. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_mm_initialization.py +0 -0
  118. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_normxcorr1d.py +0 -0
  119. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_reassignment.py +0 -0
  120. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_relocation.py +0 -0
  121. {dartsort-0.5.9 → dartsort-0.5.11}/notebook/thomas.py +0 -0
  122. {dartsort-0.5.9 → dartsort-0.5.11}/pyproject.toml +0 -0
  123. {dartsort-0.5.9 → dartsort-0.5.11}/setup.cfg +0 -0
  124. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/__init__.py +0 -0
  125. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/cli.py +0 -0
  126. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/__init__.py +0 -0
  127. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/density.py +0 -0
  128. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/forward_backward.py +0 -0
  129. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/merge.py +0 -0
  130. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/modes.py +0 -0
  131. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/detect/__init__.py +0 -0
  132. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/detect/detect.py +0 -0
  133. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/detect/detect_filters.py +0 -0
  134. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/__init__.py +0 -0
  135. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/hybrid_util.py +0 -0
  136. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  137. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  138. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/simkit.py +0 -0
  139. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/simlib.py +0 -0
  140. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/localize/__init__.py +0 -0
  141. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/localize/localize_torch.py +0 -0
  142. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/localize/localize_util.py +0 -0
  143. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/__init__.py +0 -0
  144. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/__init__.py +0 -0
  145. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  146. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  147. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/shaver.py +0 -0
  148. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/threshold.py +0 -0
  149. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  150. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/detect_np1.pt +0 -0
  151. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/detect_np2.pt +0 -0
  152. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  153. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  154. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  155. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  156. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  157. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  158. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/__init__.py +0 -0
  159. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/realignment.py +0 -0
  160. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/__init__.py +0 -0
  161. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/all_transformers.py +0 -0
  162. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/amplitudes.py +0 -0
  163. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/denoising_scorer.py +0 -0
  164. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/enforce_decrease.py +0 -0
  165. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/fixed_prop.py +0 -0
  166. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/interp.py +0 -0
  167. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/localize.py +0 -0
  168. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/matching_denoiser.py +0 -0
  169. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/reduction.py +0 -0
  170. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  171. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/supervised_denoiser.py +0 -0
  172. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/__init__.py +0 -0
  173. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/drift_util.py +0 -0
  174. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/interpolation_util.py +0 -0
  175. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/job_util.py +0 -0
  176. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/logging_util.py +0 -0
  177. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/more_operators.py +0 -0
  178. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/motion.py +0 -0
  179. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/multiprocessing_util.py +0 -0
  180. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/nn_util.py +0 -0
  181. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/peel_util.py +0 -0
  182. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/preprocess_util.py +0 -0
  183. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/py_util.py +0 -0
  184. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/registration_util.py +0 -0
  185. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/residual_util.py +0 -0
  186. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/sparse_util.py +0 -0
  187. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/spikeio.py +0 -0
  188. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/testing_util/__init__.py +0 -0
  189. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  190. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/torch_optimization_util.py +0 -0
  191. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/torch_util.py +0 -0
  192. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/waveform_util.py +0 -0
  193. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/__init__.py +0 -0
  194. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/colors.py +0 -0
  195. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/layout.py +0 -0
  196. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/metrics_util.py +0 -0
  197. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/recanim.py +0 -0
  198. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/scatterplots.py +0 -0
  199. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/versus.py +0 -0
  200. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/SOURCES.txt +0 -0
  201. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/dependency_links.txt +0 -0
  202. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/entry_points.txt +0 -0
  203. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/requires.txt +0 -0
  204. {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/top_level.txt +0 -0
  205. {dartsort-0.5.9 → dartsort-0.5.11}/tests/conftest.py +0 -0
  206. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_alignment.py +0 -0
  207. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_cluster_util.py +0 -0
  208. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_data_util.py +0 -0
  209. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_density.py +0 -0
  210. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_detect.py +0 -0
  211. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_drift_util.py +0 -0
  212. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_drifty_matching.py +0 -0
  213. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_enforce_decrease.py +0 -0
  214. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_grab_and_featurize.py +0 -0
  215. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_linalg.py +0 -0
  216. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_localize_torch.py +0 -0
  217. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_mixture_classifier.py +0 -0
  218. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_modeling.py +0 -0
  219. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_simkit.py +0 -0
  220. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_spiketorch.py +0 -0
  221. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_templates.py +0 -0
  222. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_threshold.py +0 -0
  223. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_torch_optimization_util.py +0 -0
  224. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_transform.py +0 -0
  225. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_util.py +0 -0
  226. {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_waveform_util.py +0 -0
  227. {dartsort-0.5.9 → dartsort-0.5.11}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.9
3
+ Version: 0.5.11
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -75,7 +75,7 @@ $ pip install dartsort
75
75
 
76
76
  If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
77
77
 
78
- ## Setting up a Python environment
78
+ ### Setting up a Python environment
79
79
 
80
80
  If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
81
81
 
@@ -32,7 +32,7 @@ $ pip install dartsort
32
32
 
33
33
  If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
34
34
 
35
- ## Setting up a Python environment
35
+ ### Setting up a Python environment
36
36
 
37
37
  If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
38
38
 
@@ -32,7 +32,7 @@ $ pip install dartsort
32
32
 
33
33
  If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
34
34
 
35
- ## Setting up a Python environment
35
+ ### Setting up a Python environment
36
36
 
37
37
  If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
38
38
 
@@ -39,7 +39,6 @@ from .cluster_util import (
39
39
  reorder_by_depth,
40
40
  sparsify_labels,
41
41
  )
42
- from .mixture import Scores
43
42
 
44
43
  logger = get_logger(__name__)
45
44
 
@@ -56,7 +55,7 @@ class Agglomeration:
56
55
  def agglomerate(
57
56
  *,
58
57
  sorting: DARTsortSorting,
59
- recording: BaseRecording | None,
58
+ recording: BaseRecording,
60
59
  template_merge_cfg: TemplateMergeConfig | None,
61
60
  refinement_cfg: RefinementConfig | None,
62
61
  motion: MotionInfo,
@@ -104,7 +103,7 @@ def agglomerate(
104
103
  )
105
104
 
106
105
  # tdist tells us the possible merges
107
- mask = linkage_mask(
106
+ distance_mask = linkage_mask(
108
107
  tdist.distances,
109
108
  linkage_method=template_merge_cfg.linkage,
110
109
  threshold=template_merge_cfg.merge_distance_threshold,
@@ -117,9 +116,9 @@ def agglomerate(
117
116
  dt=refinement_cfg.glom_firing_corr_dt,
118
117
  method=refinement_cfg.glom_firing_corr_method,
119
118
  )
120
- _oldsum = mask[np.triu_indices_from(mask)].sum()
119
+ _oldsum = distance_mask[np.triu_indices_from(distance_mask)].sum()
121
120
  fcorr_mask = fcorr <= refinement_cfg.glom_max_firing_corr
122
- mask = np.logical_and(mask, fcorr_mask)
121
+ mask = np.logical_and(distance_mask, fcorr_mask)
123
122
  np.fill_diagonal(mask, True)
124
123
  _newsum = mask[np.triu_indices_from(mask)].sum()
125
124
  logger.dartsortdebug(
@@ -127,6 +126,7 @@ def agglomerate(
127
126
  )
128
127
  else:
129
128
  fcorr = fcorr_mask = None
129
+ mask = distance_mask
130
130
 
131
131
  # restrict mask by overlap criteria
132
132
  qda_res = qda(
@@ -158,22 +158,51 @@ def agglomerate(
158
158
  assert np.all(qda_mask <= mask)
159
159
  if fcorr_mask is not None:
160
160
  assert np.all(np.logical_and(qda_mask, fcorr_mask) <= mask)
161
+
162
+ simg = refinement_cfg.spikeinterface_merge_preset
163
+ if simg is not None and simg != "none":
164
+ pair_mask = tdist.distances < refinement_cfg.spikeinterface_merge_max_distance
165
+ if refinement_cfg.spikeinterface_merge_min_coentropy is not None:
166
+ cmask, _ = coentropy_merge_mask(
167
+ sorting=sorting,
168
+ min_coentropy=refinement_cfg.spikeinterface_merge_min_coentropy,
169
+ coverage_threshold=refinement_cfg.spikeinterface_merge_coent_coverage,
170
+ iou_threshold=refinement_cfg.spikeinterface_merge_coent_iou,
171
+ )
172
+ pair_mask = np.logical_or(cmask, pair_mask)
173
+
174
+ si_mask = spikeinterface_merge_mask(
175
+ recording=recording,
176
+ sorting=sorting,
177
+ preset=refinement_cfg.spikeinterface_merge_preset,
178
+ censor_ms=refinement_cfg.censor_ms,
179
+ template_data=tdist.template_data,
180
+ pair_mask=pair_mask,
181
+ )
182
+ else:
183
+ si_mask = None
184
+
185
+ # force merges for very close neighbors
161
186
  force_mask = linkage_mask(
162
187
  tdist.distances,
163
188
  linkage_method=template_merge_cfg.linkage,
164
189
  threshold=refinement_cfg.qda_force_merge_for_temp_dist_below,
165
190
  )
166
- qda_mask = np.logical_or(qda_mask, force_mask)
167
- np.fill_diagonal(qda_mask, True)
168
- qda_as_dist = np.logical_not(qda_mask).astype(np.float32)
191
+
192
+ # extract final mask
193
+ final_mask = np.logical_or(qda_mask, force_mask)
194
+ if si_mask is not None:
195
+ final_mask = np.logical_or(final_mask, si_mask)
196
+ np.fill_diagonal(final_mask, True)
197
+ final_mask_as_distance = np.logical_not(final_mask).astype(np.float32)
169
198
 
170
199
  agg_sorting, new_ids = recluster(
171
200
  sorting=sorting,
172
201
  unit_ids=tdist.template_data.unit_ids,
173
- dists=qda_as_dist,
202
+ dists=final_mask_as_distance,
174
203
  shifts=tdist.shifts,
175
204
  unit_snrs=tdist.template_data.snrs_by_channel().max(1),
176
- threshold=0.5,
205
+ threshold=0.5, # binary input here
177
206
  link=template_merge_cfg.linkage,
178
207
  )
179
208
 
@@ -331,25 +360,103 @@ def template_distances(
331
360
  )
332
361
 
333
362
 
334
- def _get_scores(sorting: DARTsortSorting) -> tuple[np.ndarray, Scores]:
335
- cand = getattr(sorting, "gmm_candidates", None)
336
- log_liks = getattr(sorting, "gmm_log_liks", None)
337
- resp = getattr(sorting, "gmm_responsibilities", None)
363
+ def spikeinterface_merge_mask(
364
+ *,
365
+ recording: BaseRecording,
366
+ sorting: DARTsortSorting,
367
+ preset: str | None,
368
+ censor_ms: float = 0.0,
369
+ template_data: TemplateData,
370
+ pair_mask: np.ndarray,
371
+ min_count: int = 100,
372
+ ):
373
+ from spikeinterface.curation.auto_merge import compute_merge_unit_groups
374
+ from spikeinterface.postprocessing import ComputeTemplateSimilarity
338
375
 
339
- assert cand is not None
340
- assert log_liks is not None
341
- assert resp is not None
376
+ # censor first
377
+ if censor_ms:
378
+ sorting = deduplicate_spikes(sorting, censor_ms)
342
379
 
343
- cand = torch.asarray(cand)
344
- log_liks = torch.asarray(log_liks)
345
- resp = torch.asarray(resp)
380
+ # analyzer (lightweight one)
381
+ analyzer = sorting.to_sorting_analyzer(
382
+ recording=recording,
383
+ template_data=template_data,
384
+ compute_extensions=None,
385
+ compute_extensions_if_templates=None,
386
+ estimate_si_sparsity=False,
387
+ compute_template_similarity=False,
388
+ )
346
389
 
347
- scores = Scores(
348
- candidates=cand, log_liks=log_liks, responsibilities=resp, duties=None
390
+ # register the mask as the template similarity extension
391
+ tsim_ext = ComputeTemplateSimilarity(analyzer)
392
+ tsim_ext.data = {"similarity": pair_mask.astype(np.float32)}
393
+ tsim_ext.params = {"method": "dartsort"}
394
+ tsim_ext.run_info = {"run_completed": True}
395
+ analyzer.extensions["template_similarity"] = tsim_ext
396
+
397
+ # handle custom presets
398
+ if preset == "dartsort_slay_xc":
399
+ steps = [
400
+ "num_spikes",
401
+ "remove_contaminated",
402
+ "unit_locations",
403
+ "template_similarity",
404
+ "slay_score",
405
+ "cross_contamination",
406
+ "quality_score",
407
+ ]
408
+ preset = None
409
+ analyzer.compute_one_extension("correlograms")
410
+ elif preset == "dartsort_slay_ccg":
411
+ steps = [
412
+ "num_spikes",
413
+ "remove_contaminated",
414
+ "unit_locations",
415
+ "template_similarity",
416
+ "correlogram",
417
+ "slay_score",
418
+ "quality_score",
419
+ ]
420
+ preset = None
421
+ analyzer.compute_one_extension("correlograms")
422
+ elif preset == "dartsort_slay_xc_ccg":
423
+ steps = [
424
+ "num_spikes",
425
+ "remove_contaminated",
426
+ "unit_locations",
427
+ "template_similarity",
428
+ "correlogram",
429
+ "cross_contamination",
430
+ "slay_score",
431
+ "quality_score",
432
+ ]
433
+ preset = None
434
+ analyzer.compute_one_extension("correlograms")
435
+ else:
436
+ assert preset is not None
437
+ steps = None
438
+
439
+ # make parameters aware of censorship and other params
440
+ my_step_params = {
441
+ "num_spikes": {"min_spikes": min_count},
442
+ "remove_contaminated": {"censored_period_ms": censor_ms},
443
+ "template_similarity": {"similarity_method": "dartsort"},
444
+ "correlogram": {"censor_correlograms_ms": censor_ms},
445
+ "cross_contamination": {"censored_period_ms": censor_ms},
446
+ "quality_score": {"censored_period_ms": censor_ms},
447
+ }
448
+ groups = compute_merge_unit_groups(
449
+ preset=preset,
450
+ steps=steps,
451
+ sorting_analyzer=analyzer,
452
+ steps_params=my_step_params,
453
+ force_copy=False,
349
454
  )
350
- assert sorting.labels is not None
351
- labels = sorting.labels
352
- return labels, scores
455
+ mask = np.zeros_like(pair_mask)
456
+ for g in groups:
457
+ g = np.array(g)
458
+ mask[g[:, None], g[None, :]] = True
459
+ return mask
353
460
 
354
461
 
355
462
  @databag
@@ -392,8 +499,12 @@ def qda(
392
499
  show_progress: bool,
393
500
  computation_cfg: ComputationConfig,
394
501
  ) -> QDAResult:
502
+ from ..util.data_util import get_gmm_scores
503
+
395
504
  # reconstruct scores from sorting attached data (exclude train_ix?)
396
- glabels, gscores = _get_scores(sorting)
505
+ gscores = get_gmm_scores(sorting)
506
+ glabels = sorting.labels
507
+ assert glabels is not None
397
508
 
398
509
  if mask is None:
399
510
  mask = np.ones((sorting.n_units, sorting.n_units), dtype=bool)
@@ -665,7 +776,8 @@ def combine_gmm_scores(
665
776
 
666
777
  # check invariants at the top
667
778
  if responsibilities.shape[1] > 2:
668
- assert np.all(np.diff(responsibilities[:, :-1], axis=1) <= 0)
779
+ _maxdiff = np.diff(responsibilities[:, :-1], axis=1).max()
780
+ assert _maxdiff <= 1e-3, _maxdiff
669
781
  assert np.greater_equal(np.isneginf(logliks[:, :-1]), candidates == -1).all()
670
782
  if sorting.labels is not None:
671
783
  assert np.all(
@@ -698,7 +810,8 @@ def combine_gmm_scores(
698
810
 
699
811
  # check invariants at the bottom
700
812
  if mergedr.shape[1] > 2:
701
- assert np.all(np.diff(mergedr[:, : cand.shape[1]], axis=1) <= 0)
813
+ _maxdiff = np.diff(mergedr[:, : cand.shape[1]], axis=1).max()
814
+ assert _maxdiff <= 1e-3, _maxdiff
702
815
  assert np.greater_equal(np.isneginf(mergedl[:, : cand.shape[1]]), cand == -1).all()
703
816
  assert (cand < 0).sum() >= nbye
704
817
  if sorting.labels is not None:
@@ -733,7 +846,7 @@ def _combine_loop(
733
846
  continue
734
847
 
735
848
  eq_ncandj = rcand[j + 1 :] == ncandj
736
- if eq_ncandj.sum() <= 1:
849
+ if eq_ncandj.sum() < 1:
737
850
  continue
738
851
 
739
852
  rsum = mergedr[s, j]
@@ -758,7 +871,7 @@ def _combine_loop(
758
871
  def deduplicate_spikes(
759
872
  sorting: DARTsortSorting,
760
873
  radius_ms: float = -1.0,
761
- score_by=("merged_logliks", "gmm_logliks", "scores"),
874
+ score_by=("merged_log_liks", "gmm_log_liks", "scores"),
762
875
  ) -> DARTsortSorting:
763
876
  """The lower-scoring of any spikes within radius_samples of each other is relabeled to -1.
764
877
 
@@ -783,7 +896,7 @@ def deduplicate_spikes(
783
896
  break
784
897
  if scores is None:
785
898
  raise ValueError(f"sorting had none of {score_by}.")
786
- if scores.ndim == 2:
899
+ if scores.ndim >= 2:
787
900
  scores = scores[:, 0]
788
901
  assert scores.ndim == 1
789
902
  assert scores.shape == new_labels.shape
@@ -799,6 +912,8 @@ def deduplicate_spikes(
799
912
  ndrop = 0
800
913
  for unit_id in unit_ids:
801
914
  in_unit = np.flatnonzero(new_labels == unit_id)
915
+ if in_unit.size <= 1:
916
+ continue
802
917
  t = times_samples[in_unit]
803
918
  dt = np.diff(t)
804
919
  if dt.min() > radius_samples:
@@ -869,3 +984,152 @@ def _dedup_unit_loop(
869
984
  break
870
985
 
871
986
  i0 = i1
987
+
988
+
989
+ @databag
990
+ class CoentropyResult:
991
+ coentropy: np.ndarray
992
+ """KxK; reduction of entropy per cooccurrence due to merging pair"""
993
+
994
+ cooccurrence: np.ndarray
995
+ """KxK; number of times these units score the same spike"""
996
+
997
+ rival_count: np.ndarray
998
+ """KxK; number of times one unit scores a spike where the other is top"""
999
+
1000
+ occurrence: np.ndarray
1001
+ """K; number of times the unit appears in the candidates at all"""
1002
+
1003
+ cov: np.ndarray
1004
+ """KxK; rival count / max pair count (rival diag)"""
1005
+
1006
+ iou: np.ndarray
1007
+ """KxK; rival count over pair sum"""
1008
+
1009
+
1010
+ def coentropy_merge_mask(
1011
+ sorting: DARTsortSorting,
1012
+ min_coentropy: float,
1013
+ coverage_threshold: float,
1014
+ iou_threshold: float,
1015
+ gmm_prefix=("merged", "gmm"),
1016
+ ) -> tuple[np.ndarray, CoentropyResult]:
1017
+ """
1018
+ Parameters
1019
+ ----------
1020
+ sorting : DARTsortSorting
1021
+ min_coentropy : float
1022
+ Must be met by pair for mask=True
1023
+ min_coverage : float
1024
+ Pairs such that at least one unit in each pair has
1025
+ rival_count/count > mincov are allowed
1026
+ iou_threshold: float
1027
+ Pairs with rival iou > iouthresh are allowed
1028
+ """
1029
+ c = coentropy(sorting, gmm_prefix=gmm_prefix)
1030
+ assert c is not None
1031
+
1032
+ mask = np.logical_or(c.cov >= coverage_threshold, c.iou >= iou_threshold)
1033
+ mask = np.logical_and(c.coentropy >= min_coentropy, mask)
1034
+ np.fill_diagonal(mask, True)
1035
+ return mask, c
1036
+
1037
+
1038
+ def coentropy(
1039
+ sorting: DARTsortSorting,
1040
+ gmm_prefix=("merged", "gmm"),
1041
+ ) -> CoentropyResult | None:
1042
+ """Calculate entropy reduction due to merging pairs."""
1043
+ for k in gmm_prefix:
1044
+ cands = getattr(sorting, f"{k}_candidates", None)
1045
+ resps = getattr(sorting, f"{k}_responsibilities", None)
1046
+ if cands is not None:
1047
+ assert resps is not None
1048
+ break
1049
+ else:
1050
+ return None
1051
+
1052
+ k = sorting.n_units
1053
+ resps = resps[:, : cands.shape[1]].astype(np.float64)
1054
+ coentropy = np.zeros((k, k))
1055
+ cooccurrence = np.zeros((k, k), dtype=np.int64)
1056
+ rival_count = np.zeros((k, k), dtype=np.int64)
1057
+ occurrence = np.zeros((k,), dtype=np.int64)
1058
+ _calc_coentropy(coentropy, cooccurrence, rival_count, occurrence, cands, resps)
1059
+ rival_count += rival_count.T
1060
+ cdiag = np.diagonal(rival_count)
1061
+ assert (cdiag % 2 == 0).all()
1062
+ np.fill_diagonal(rival_count, cdiag // 2)
1063
+ coentropy += coentropy.T
1064
+ cooccurrence += cooccurrence.T
1065
+
1066
+ # rival count diagonal is just unit top count (not exactly label count,
1067
+ # since it doesn't account for noise assignments)
1068
+ counts = np.diagonal(rival_count)
1069
+ counts = np.maximum(counts, 1)
1070
+
1071
+ cov = rival_count / counts
1072
+ cov = np.minimum(cov, cov.T)
1073
+
1074
+ # this is a disjoint union, since it's the top-label count
1075
+ union = counts[:, None] + counts[None, :]
1076
+ iou = rival_count / union
1077
+
1078
+ return CoentropyResult(
1079
+ coentropy=coentropy,
1080
+ cooccurrence=cooccurrence,
1081
+ rival_count=rival_count,
1082
+ occurrence=occurrence,
1083
+ cov=cov,
1084
+ iou=iou,
1085
+ )
1086
+
1087
+
1088
+ @numba.njit(parallel=True)
1089
+ def _calc_coentropy(
1090
+ coentropy: np.ndarray,
1091
+ cooccurrence: np.ndarray,
1092
+ rival_count: np.ndarray,
1093
+ occurrence: np.ndarray,
1094
+ cands: np.ndarray,
1095
+ resps: np.ndarray,
1096
+ ):
1097
+ for i in numba.prange(cands.shape[0]): # ty: ignore
1098
+ u = cands[i]
1099
+ q = resps[i]
1100
+ log_q = np.log(q)
1101
+ np.nan_to_num(log_q, copy=False, neginf=0.0)
1102
+ dh = q * log_q
1103
+
1104
+ ui0 = u[0]
1105
+ qi0 = q[0]
1106
+ dhi0 = dh[0]
1107
+
1108
+ occurrence[ui0] += 1
1109
+ rival_count[ui0, ui0] += 1
1110
+
1111
+ for j in range(1, cands.shape[1]):
1112
+ uj = u[j]
1113
+ if uj < 0:
1114
+ break
1115
+
1116
+ ii = min(ui0, uj)
1117
+ jj = max(ui0, uj)
1118
+
1119
+ occurrence[uj] += 1
1120
+ rival_count[ui0, uj] += 1
1121
+
1122
+ cij = cooccurrence[ii, jj] + 1
1123
+ cooccurrence[ii, jj] = cij
1124
+
1125
+ # change in entropy due to merging uj, uk:
1126
+ # subtract their current contribution, add the new contribution
1127
+ # we want reduction of entropy, so this is the negative of that!
1128
+ qij = q[j] + qi0
1129
+ dhij = dh[j] + dhi0
1130
+ if qij > 0:
1131
+ dhij -= qij * np.log(qij)
1132
+
1133
+ # Welford mean of -dh
1134
+ cur_coent = coentropy[ii, jj]
1135
+ coentropy[ii, jj] = cur_coent + (-dhij - cur_coent) / cij
@@ -170,10 +170,11 @@ def linkage_mask(
170
170
  return mask
171
171
 
172
172
 
173
- def sparsify_labels(labels: np.ndarray) -> dict[int, np.ndarray]:
173
+ def sparsify_labels(labels: np.ndarray, ids: np.ndarray | None = None) -> dict[int, np.ndarray]:
174
174
  assert labels.ndim == 1
175
- ids = np.unique(labels)
176
- ids = ids[ids >= 0]
175
+ if ids is None:
176
+ ids = np.unique(labels)
177
+ ids = ids[ids >= 0]
177
178
  inj = {}
178
179
  for j in ids:
179
180
  inj[j] = np.flatnonzero(labels == j)
@@ -19,7 +19,16 @@ from ..util.job_util import ensure_computation_config
19
19
  from ..util.main_util import ds_save_intermediate_labels
20
20
  from ..util.motion import MotionInfo
21
21
  from ..util.multiprocessing_util import handle_negative_jobs
22
- from . import agglomerate, cluster_util, density, forward_backward, mixture, refine_util
22
+ from ..util.torch_util import cleanup_and_log_gpu_usage
23
+ from . import (
24
+ agglomerate,
25
+ cluster_util,
26
+ density,
27
+ forward_backward,
28
+ kmeans,
29
+ mixture,
30
+ refine_util,
31
+ )
23
32
  from .clustering_features import SimpleMatrixFeatures, StableWaveformFeatures
24
33
 
25
34
  if TYPE_CHECKING:
@@ -292,6 +301,10 @@ class DensityPeaksClusterer(Clusterer):
292
301
  workers=-1,
293
302
  uhdversion=False,
294
303
  random_seed=0,
304
+ kmeans_cleanup=True,
305
+ kmeans_max_sigma: float = 5.0,
306
+ kmeans_iter=100,
307
+ device=None,
295
308
  **kwargs,
296
309
  ):
297
310
  super().__init__(**kwargs)
@@ -308,6 +321,10 @@ class DensityPeaksClusterer(Clusterer):
308
321
  self.workers = workers
309
322
  self.uhdversion = uhdversion
310
323
  self.random_seed = random_seed
324
+ self.kmeans_cleanup = kmeans_cleanup
325
+ self.kmeans_max_sigma = kmeans_max_sigma
326
+ self.kmeans_iter = kmeans_iter
327
+ self.device = device
311
328
 
312
329
  @classmethod
313
330
  def from_config(
@@ -335,6 +352,7 @@ class DensityPeaksClusterer(Clusterer):
335
352
  outlier_radius=clustering_cfg.outlier_radius,
336
353
  outlier_neighbor_count=clustering_cfg.outlier_neighbor_count,
337
354
  workers=workers,
355
+ device=computation_cfg.actual_device(),
338
356
  uhdversion=uhdversion,
339
357
  computation_cfg=computation_cfg,
340
358
  waveform_cfg=waveform_cfg,
@@ -342,6 +360,9 @@ class DensityPeaksClusterer(Clusterer):
342
360
  save_labels_dir=save_labels_dir,
343
361
  labels_fmt=labels_fmt,
344
362
  sampling_cfg=clustering_cfg.sampling_cfg,
363
+ kmeans_cleanup=clustering_cfg.dpc_kmeans_cleanup,
364
+ kmeans_iter=clustering_cfg.kmeans_iter,
365
+ kmeans_max_sigma=clustering_cfg.gmmdpc_max_sigma,
345
366
  )
346
367
 
347
368
  def _cluster(
@@ -425,6 +446,19 @@ class DensityPeaksClusterer(Clusterer):
425
446
  kdtree = None
426
447
  labels = res["labels"]
427
448
 
449
+ if self.kmeans_cleanup:
450
+ kres = kmeans.truncated_kmeans_from_labels(
451
+ X=X,
452
+ labels=labels,
453
+ device=self.device,
454
+ max_sigma=self.kmeans_max_sigma,
455
+ n_iter=self.kmeans_iter,
456
+ )
457
+ assert kres.labels is not None
458
+ labels = kres.labels.numpy(force=True)
459
+ del kres
460
+ cleanup_and_log_gpu_usage(self.computation_cfg, "DPC->kmeans")
461
+
428
462
  labels = cluster_util.decrumb(
429
463
  labels, min_size=self.remove_clusters_smaller_than, in_place=True
430
464
  )
@@ -821,6 +855,7 @@ class AgglomerateRefinement(Refinement):
821
855
  recording: BaseRecording | None,
822
856
  motion: MotionInfo,
823
857
  ):
858
+ assert recording is not None
824
859
  return agglomerate.agglomerate(
825
860
  recording=recording,
826
861
  sorting=sorting,
@@ -835,6 +870,48 @@ class AgglomerateRefinement(Refinement):
835
870
  refinement_strategies["agglomerate"] = AgglomerateRefinement
836
871
 
837
872
 
873
+ class FilterRefinement(Refinement):
874
+ """Runs various filters as specified in cfg"""
875
+
876
+ def needs_stable_features(self):
877
+ return super().needs_stable_features() or bool(
878
+ self.refinement_cfg.collision_cleaning_error_threshold
879
+ )
880
+
881
+ def _refine(
882
+ self,
883
+ features: SimpleMatrixFeatures,
884
+ stable_features: StableWaveformFeatures | None,
885
+ sorting: DARTsortSorting,
886
+ recording: BaseRecording | None,
887
+ motion: MotionInfo,
888
+ ):
889
+ assert recording is not None
890
+
891
+ if self.refinement_cfg.collision_cleaning_error_threshold:
892
+ assert stable_features is not None
893
+ sorting = refine_util.collision_cleaning_error_filter(
894
+ sorting=sorting,
895
+ recording=recording,
896
+ stable_features=stable_features,
897
+ refinement_cfg=self.refinement_cfg,
898
+ motion=motion,
899
+ computation_cfg=self.computation_cfg,
900
+ ).sorting
901
+
902
+ if self.refinement_cfg.gmm_isolation_threshold:
903
+ sorting = refine_util.gmm_isolation_filter(
904
+ sorting=sorting,
905
+ refinement_cfg=self.refinement_cfg,
906
+ computation_cfg=self.computation_cfg,
907
+ ).sorting
908
+
909
+ return sorting
910
+
911
+
912
+ refinement_strategies["filter"] = FilterRefinement
913
+
914
+
838
915
  class ForwardBackwardEnsembler(Refinement):
839
916
  """If there are more time chunk ones, make a new ABC with this logic."""
840
917