DARTsort 0.5.8__tar.gz → 0.5.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (226) hide show
  1. {dartsort-0.5.8 → dartsort-0.5.10}/PKG-INFO +3 -3
  2. {dartsort-0.5.8 → dartsort-0.5.10}/README.md +1 -1
  3. {dartsort-0.5.8 → dartsort-0.5.10}/docs/index.md +1 -1
  4. {dartsort-0.5.8 → dartsort-0.5.10}/pyproject.toml +1 -1
  5. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/agglomerate.py +290 -15
  6. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/clustering.py +36 -1
  7. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/clustering_features.py +37 -12
  8. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/kmeans.py +309 -11
  9. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/mixture.py +200 -67
  10. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/config.py +13 -6
  11. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/analysis.py +19 -7
  12. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/comparison.py +33 -4
  13. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/main.py +20 -12
  14. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/grab.py +89 -16
  15. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching.py +24 -6
  16. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/compressed_upsampled.py +12 -1
  17. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/drifty.py +82 -40
  18. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/matching_base.py +12 -7
  19. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/peel_base.py +55 -16
  20. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/peel_lib.py +63 -29
  21. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/reduction_template.py +14 -9
  22. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/subtract.py +51 -10
  23. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/get_templates.py +2 -2
  24. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/postprocess_util.py +4 -4
  25. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/realignment.py +2 -1
  26. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/template_util.py +7 -3
  27. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/templates.py +10 -4
  28. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/templib.py +5 -0
  29. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/_multichan_denoiser_kit.py +12 -2
  30. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/decollider.py +62 -12
  31. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/mixture_classifier.py +3 -2
  32. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/pipeline.py +7 -1
  33. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/temporal_pca.py +39 -14
  34. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/transform_base.py +22 -3
  35. dartsort-0.5.10/src/dartsort/transform/whiten.py +102 -0
  36. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/data_util.py +171 -1
  37. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/internal_config.py +81 -31
  38. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/interpolation_util.py +5 -5
  39. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/main_util.py +6 -4
  40. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/motion.py +6 -4
  41. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/noise_util.py +180 -27
  42. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/spiketorch.py +68 -18
  43. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/testing_util/matching_debug_util.py +40 -6
  44. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/torch_util.py +46 -3
  45. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/analysis_plots.py +166 -5
  46. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/gt.py +0 -3
  47. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/mixture.py +6 -6
  48. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/unit.py +174 -71
  49. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/unit_comparison.py +12 -6
  50. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/PKG-INFO +3 -3
  51. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/requires.txt +1 -1
  52. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_config.py +2 -1
  53. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_kmeans.py +7 -4
  54. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_matching.py +2 -0
  55. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_subtract.py +8 -9
  56. dartsort-0.5.8/src/dartsort/transform/whiten.py +0 -34
  57. {dartsort-0.5.8 → dartsort-0.5.10}/.github/workflows/ci.yml +0 -0
  58. {dartsort-0.5.8 → dartsort-0.5.10}/.github/workflows/ci_windows.yml +0 -0
  59. {dartsort-0.5.8 → dartsort-0.5.10}/.github/workflows/deploy.yml +0 -0
  60. {dartsort-0.5.8 → dartsort-0.5.10}/.github/workflows/docs.yml +0 -0
  61. {dartsort-0.5.8 → dartsort-0.5.10}/.gitignore +0 -0
  62. {dartsort-0.5.8 → dartsort-0.5.10}/docs/main_api.md +0 -0
  63. {dartsort-0.5.8 → dartsort-0.5.10}/docs/toolkit_api.md +0 -0
  64. {dartsort-0.5.8 → dartsort-0.5.10}/environment.yml +0 -0
  65. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/.keep +0 -0
  66. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/basic_integration_test.py +0 -0
  67. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/ci diag.py +0 -0
  68. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/dartsort_dredge_demo.ipynb +0 -0
  69. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/dartsort_dredge_demo.py +0 -0
  70. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/data test.py +0 -0
  71. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/dead reckoning.py +0 -0
  72. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/denoise np1.py +0 -0
  73. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/denoise np2.py +0 -0
  74. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/exploratory.py +0 -0
  75. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/geom debug.py +0 -0
  76. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/hdbscan_clustering_5min.py +0 -0
  77. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/hybrid 10.py +0 -0
  78. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/hybrid cluster.py +0 -0
  79. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/hybrid_11.py +0 -0
  80. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/initial training.py +0 -0
  81. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/make hybrid.py +0 -0
  82. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/mgh.py +0 -0
  83. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/np1 feats.py +0 -0
  84. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/np2 processing version c.py +0 -0
  85. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/paperfigs.py +0 -0
  86. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/pdists.py +0 -0
  87. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/postcheck.py +0 -0
  88. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/ptp vae.py +0 -0
  89. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/re clust.py +0 -0
  90. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/reg fig loop.py +0 -0
  91. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/reg_sim_exp.py +0 -0
  92. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/relocating.py +0 -0
  93. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/relocating_all.py +0 -0
  94. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/restats +0 -0
  95. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/sortStinger1.py +0 -0
  96. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/sortStinger3.py +0 -0
  97. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/sortStinger4.py +0 -0
  98. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/sortStinger5.py +0 -0
  99. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/sortStinger6.py +0 -0
  100. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/spike farm.py +0 -0
  101. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/ss_ensemble_sim.py +0 -0
  102. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/ss_ensemble_sim_drift.py +0 -0
  103. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/ss_sim.py +0 -0
  104. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/stan diagnostics.py +0 -0
  105. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/storytelling c.py +0 -0
  106. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/storytelling.py +0 -0
  107. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
  108. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/debug_matching.py +0 -0
  109. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_agreement.py +0 -0
  110. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_amplitude_scaling.py +0 -0
  111. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_batch_optim.py +0 -0
  112. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_ibme.py +0 -0
  113. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_mm_initialization.py +0 -0
  114. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_normxcorr1d.py +0 -0
  115. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_reassignment.py +0 -0
  116. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_relocation.py +0 -0
  117. {dartsort-0.5.8 → dartsort-0.5.10}/notebook/thomas.py +0 -0
  118. {dartsort-0.5.8 → dartsort-0.5.10}/setup.cfg +0 -0
  119. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/__init__.py +0 -0
  120. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/cli.py +0 -0
  121. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/__init__.py +0 -0
  122. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/cluster_util.py +0 -0
  123. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/density.py +0 -0
  124. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/forward_backward.py +0 -0
  125. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/merge.py +0 -0
  126. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/modes.py +0 -0
  127. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/refine_util.py +0 -0
  128. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/detect/__init__.py +0 -0
  129. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/detect/detect.py +0 -0
  130. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/detect/detect_filters.py +0 -0
  131. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/__init__.py +0 -0
  132. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/hybrid_util.py +0 -0
  133. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  134. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  135. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/simkit.py +0 -0
  136. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/simlib.py +0 -0
  137. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/localize/__init__.py +0 -0
  138. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/localize/localize_torch.py +0 -0
  139. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/localize/localize_util.py +0 -0
  140. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/__init__.py +0 -0
  141. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/__init__.py +0 -0
  142. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  143. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  144. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/shaver.py +0 -0
  145. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/threshold.py +0 -0
  146. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  147. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/detect_np1.pt +0 -0
  148. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/detect_np2.pt +0 -0
  149. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  150. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  151. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  152. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  153. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  154. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  155. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/__init__.py +0 -0
  156. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/__init__.py +0 -0
  157. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/all_transformers.py +0 -0
  158. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/amortized_localization.py +0 -0
  159. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/amplitudes.py +0 -0
  160. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/denoising_scorer.py +0 -0
  161. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/enforce_decrease.py +0 -0
  162. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/fixed_prop.py +0 -0
  163. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/interp.py +0 -0
  164. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/localize.py +0 -0
  165. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/matching_denoiser.py +0 -0
  166. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/reduction.py +0 -0
  167. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  168. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/supervised_denoiser.py +0 -0
  169. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/__init__.py +0 -0
  170. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/cli_util.py +0 -0
  171. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/drift_util.py +0 -0
  172. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/job_util.py +0 -0
  173. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/logging_util.py +0 -0
  174. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/more_operators.py +0 -0
  175. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/multiprocessing_util.py +0 -0
  176. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/nn_util.py +0 -0
  177. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/peel_util.py +0 -0
  178. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/preprocess_util.py +0 -0
  179. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/py_util.py +0 -0
  180. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/registration_util.py +0 -0
  181. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/residual_util.py +0 -0
  182. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/sparse_util.py +0 -0
  183. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/spikeio.py +0 -0
  184. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/testing_util/__init__.py +0 -0
  185. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  186. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/torch_optimization_util.py +0 -0
  187. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/waveform_util.py +0 -0
  188. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/__init__.py +0 -0
  189. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/colors.py +0 -0
  190. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/layout.py +0 -0
  191. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/metrics_util.py +0 -0
  192. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/recanim.py +0 -0
  193. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/scatterplots.py +0 -0
  194. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/sorting.py +0 -0
  195. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/versus.py +0 -0
  196. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/vismain.py +0 -0
  197. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/waveforms.py +0 -0
  198. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/SOURCES.txt +0 -0
  199. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/dependency_links.txt +0 -0
  200. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/entry_points.txt +0 -0
  201. {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/top_level.txt +0 -0
  202. {dartsort-0.5.8 → dartsort-0.5.10}/tests/conftest.py +0 -0
  203. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_alignment.py +0 -0
  204. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_cluster_util.py +0 -0
  205. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_clustering.py +0 -0
  206. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_dartsort.py +0 -0
  207. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_data_util.py +0 -0
  208. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_density.py +0 -0
  209. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_detect.py +0 -0
  210. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_drift_util.py +0 -0
  211. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_drifty_matching.py +0 -0
  212. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_enforce_decrease.py +0 -0
  213. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_grab_and_featurize.py +0 -0
  214. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_linalg.py +0 -0
  215. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_localize_torch.py +0 -0
  216. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_mixture_classifier.py +0 -0
  217. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_modeling.py +0 -0
  218. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_simkit.py +0 -0
  219. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_spiketorch.py +0 -0
  220. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_templates.py +0 -0
  221. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_threshold.py +0 -0
  222. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_torch_optimization_util.py +0 -0
  223. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_transform.py +0 -0
  224. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_util.py +0 -0
  225. {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_waveform_util.py +0 -0
  226. {dartsort-0.5.8 → dartsort-0.5.10}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.8
3
+ Version: 0.5.10
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -27,7 +27,7 @@ Requires-Dist: scipy>=1.13
27
27
  Requires-Dist: scikit-learn
28
28
  Requires-Dist: spikeinterface>=0.104.0
29
29
  Requires-Dist: sympy
30
- Requires-Dist: torch>=2.0
30
+ Requires-Dist: torch>=2.2
31
31
  Requires-Dist: tqdm
32
32
  Requires-Dist: typing-extensions
33
33
  Provides-Extra: doc
@@ -75,7 +75,7 @@ $ pip install dartsort
75
75
 
76
76
  If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
77
77
 
78
- ## Setting up a Python environment
78
+ ### Setting up a Python environment
79
79
 
80
80
  If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
81
81
 
@@ -32,7 +32,7 @@ $ pip install dartsort
32
32
 
33
33
  If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
34
34
 
35
- ## Setting up a Python environment
35
+ ### Setting up a Python environment
36
36
 
37
37
  If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
38
38
 
@@ -32,7 +32,7 @@ $ pip install dartsort
32
32
 
33
33
  If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
34
34
 
35
- ## Setting up a Python environment
35
+ ### Setting up a Python environment
36
36
 
37
37
  If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
38
38
 
@@ -32,7 +32,7 @@ dependencies = [
32
32
  "scikit-learn",
33
33
  "spikeinterface>=0.104.0",
34
34
  "sympy",
35
- "torch>=2.0",
35
+ "torch>=2.2",
36
36
  "tqdm",
37
37
  "typing-extensions",
38
38
  ]
@@ -56,7 +56,7 @@ class Agglomeration:
56
56
  def agglomerate(
57
57
  *,
58
58
  sorting: DARTsortSorting,
59
- recording: BaseRecording | None,
59
+ recording: BaseRecording,
60
60
  template_merge_cfg: TemplateMergeConfig | None,
61
61
  refinement_cfg: RefinementConfig | None,
62
62
  motion: MotionInfo,
@@ -104,7 +104,7 @@ def agglomerate(
104
104
  )
105
105
 
106
106
  # tdist tells us the possible merges
107
- mask = linkage_mask(
107
+ distance_mask = linkage_mask(
108
108
  tdist.distances,
109
109
  linkage_method=template_merge_cfg.linkage,
110
110
  threshold=template_merge_cfg.merge_distance_threshold,
@@ -117,9 +117,9 @@ def agglomerate(
117
117
  dt=refinement_cfg.glom_firing_corr_dt,
118
118
  method=refinement_cfg.glom_firing_corr_method,
119
119
  )
120
- _oldsum = mask[np.triu_indices_from(mask)].sum()
120
+ _oldsum = distance_mask[np.triu_indices_from(distance_mask)].sum()
121
121
  fcorr_mask = fcorr <= refinement_cfg.glom_max_firing_corr
122
- mask = np.logical_and(mask, fcorr_mask)
122
+ mask = np.logical_and(distance_mask, fcorr_mask)
123
123
  np.fill_diagonal(mask, True)
124
124
  _newsum = mask[np.triu_indices_from(mask)].sum()
125
125
  logger.dartsortdebug(
@@ -127,6 +127,7 @@ def agglomerate(
127
127
  )
128
128
  else:
129
129
  fcorr = fcorr_mask = None
130
+ mask = distance_mask
130
131
 
131
132
  # restrict mask by overlap criteria
132
133
  qda_res = qda(
@@ -155,23 +156,50 @@ def agglomerate(
155
156
  axis=0,
156
157
  )
157
158
  qda_mask = np.logical_or(qda_mask_uni, qda_mask_bi)
159
+ assert np.all(qda_mask <= mask)
160
+ if fcorr_mask is not None:
161
+ assert np.all(np.logical_and(qda_mask, fcorr_mask) <= mask)
162
+
163
+ if refinement_cfg.spikeinterface_merge_preset is not None:
164
+ pair_mask = tdist.distances < refinement_cfg.spikeinterface_merge_max_distance
165
+ if refinement_cfg.spikeinterface_merge_min_coentropy is not None:
166
+ cmask, _ = coentropy_merge_mask(
167
+ sorting=sorting,
168
+ min_coentropy=refinement_cfg.spikeinterface_merge_min_coentropy,
169
+ coverage_threshold=refinement_cfg.spikeinterface_merge_coent_coverage,
170
+ iou_threshold=refinement_cfg.spikeinterface_merge_coent_iou,
171
+ )
172
+ pair_mask = np.logical_or(cmask, pair_mask)
173
+
174
+ si_mask = spikeinterface_merge_mask(
175
+ recording=recording,
176
+ sorting=sorting,
177
+ preset=refinement_cfg.spikeinterface_merge_preset,
178
+ censor_ms=refinement_cfg.censor_ms,
179
+ template_data=tdist.template_data,
180
+ pair_mask=pair_mask,
181
+ )
182
+ else:
183
+ si_mask = None
184
+
185
+ # force merges for very close neighbors
158
186
  force_mask = linkage_mask(
159
187
  tdist.distances,
160
188
  linkage_method=template_merge_cfg.linkage,
161
189
  threshold=refinement_cfg.qda_force_merge_for_temp_dist_below,
162
190
  )
163
- qda_mask = np.logical_or(qda_mask, force_mask)
164
- np.fill_diagonal(qda_mask, True)
165
- if fcorr_mask is None:
166
- assert np.all(qda_mask <= mask)
167
- else:
168
- assert np.all((qda_mask & fcorr_mask) <= mask)
169
- qda_as_dist = np.logical_not(qda_mask).astype(np.float32)
191
+
192
+ # extract final mask
193
+ final_mask = np.logical_or(qda_mask, force_mask)
194
+ if si_mask is not None:
195
+ final_mask = np.logical_or(final_mask, si_mask)
196
+ np.fill_diagonal(final_mask, True)
197
+ final_mask_as_distance = np.logical_not(final_mask).astype(np.float32)
170
198
 
171
199
  agg_sorting, new_ids = recluster(
172
200
  sorting=sorting,
173
201
  unit_ids=tdist.template_data.unit_ids,
174
- dists=qda_as_dist,
202
+ dists=final_mask_as_distance,
175
203
  shifts=tdist.shifts,
176
204
  unit_snrs=tdist.template_data.snrs_by_channel().max(1),
177
205
  threshold=0.5,
@@ -353,6 +381,100 @@ def _get_scores(sorting: DARTsortSorting) -> tuple[np.ndarray, Scores]:
353
381
  return labels, scores
354
382
 
355
383
 
384
+ def spikeinterface_merge_mask(
385
+ *,
386
+ recording: BaseRecording,
387
+ sorting: DARTsortSorting,
388
+ preset: str | None,
389
+ censor_ms: float = 0.0,
390
+ template_data: TemplateData,
391
+ pair_mask: np.ndarray,
392
+ min_count: int = 100,
393
+ ):
394
+ from spikeinterface.curation.auto_merge import compute_merge_unit_groups
395
+ from spikeinterface.postprocessing import ComputeTemplateSimilarity
396
+
397
+ # censor first
398
+ if censor_ms:
399
+ sorting = deduplicate_spikes(sorting, censor_ms)
400
+
401
+ # analyzer
402
+ analyzer = sorting.to_sorting_analyzer(
403
+ recording=recording, template_data=template_data
404
+ )
405
+
406
+ # register the mask as the template similarity extension
407
+ tsim_ext = ComputeTemplateSimilarity(analyzer)
408
+ tsim_ext.data = {"similarity": pair_mask.astype(np.float32)}
409
+ tsim_ext.params = {"method": "dartsort"}
410
+ tsim_ext.run_info = {"run_completed": True}
411
+ analyzer.extensions["template_similarity"] = tsim_ext
412
+
413
+ # handle custom presets
414
+ if preset == "dartsort_slay_xc":
415
+ steps = [
416
+ "num_spikes",
417
+ "remove_contaminated",
418
+ "unit_locations",
419
+ "template_similarity",
420
+ "slay_score",
421
+ "cross_contamination",
422
+ "quality_score",
423
+ ]
424
+ preset = None
425
+ analyzer.compute_one_extension("correlograms")
426
+ elif preset == "dartsort_slay_ccg":
427
+ steps = [
428
+ "num_spikes",
429
+ "remove_contaminated",
430
+ "unit_locations",
431
+ "template_similarity",
432
+ "correlogram",
433
+ "slay_score",
434
+ "quality_score",
435
+ ]
436
+ preset = None
437
+ analyzer.compute_one_extension("correlograms")
438
+ elif preset == "dartsort_slay_xc_ccg":
439
+ steps = [
440
+ "num_spikes",
441
+ "remove_contaminated",
442
+ "unit_locations",
443
+ "template_similarity",
444
+ "correlogram",
445
+ "cross_contamination",
446
+ "slay_score",
447
+ "quality_score",
448
+ ]
449
+ preset = None
450
+ analyzer.compute_one_extension("correlograms")
451
+ else:
452
+ assert preset is not None
453
+ steps = None
454
+
455
+ # make parameters aware of censorship and other params
456
+ my_step_params = {
457
+ "num_spikes": {"min_spikes": min_count},
458
+ "remove_contaminated": {"censored_period_ms": censor_ms},
459
+ "template_similarity": {"similarity_method": "dartsort"},
460
+ "correlogram": {"censor_correlograms_ms": censor_ms},
461
+ "cross_contamination": {"censored_period_ms": censor_ms},
462
+ "quality_score": {"censored_period_ms": censor_ms},
463
+ }
464
+ groups = compute_merge_unit_groups(
465
+ preset=preset,
466
+ steps=steps,
467
+ sorting_analyzer=analyzer,
468
+ steps_params=my_step_params,
469
+ force_copy=False,
470
+ )
471
+ mask = np.zeros_like(pair_mask)
472
+ for g in groups:
473
+ g = np.array(g)
474
+ mask[g[:, None], g[None, :]] = True
475
+ return mask
476
+
477
+
356
478
  @databag
357
479
  class QDAResult:
358
480
  """Unit pair QDA metrics
@@ -666,7 +788,8 @@ def combine_gmm_scores(
666
788
 
667
789
  # check invariants at the top
668
790
  if responsibilities.shape[1] > 2:
669
- assert np.all(np.diff(responsibilities[:, :-1], axis=1) <= 0)
791
+ _maxdiff = np.diff(responsibilities[:, :-1], axis=1).max()
792
+ assert _maxdiff <= 1e-3, _maxdiff
670
793
  assert np.greater_equal(np.isneginf(logliks[:, :-1]), candidates == -1).all()
671
794
  if sorting.labels is not None:
672
795
  assert np.all(
@@ -699,7 +822,8 @@ def combine_gmm_scores(
699
822
 
700
823
  # check invariants at the bottom
701
824
  if mergedr.shape[1] > 2:
702
- assert np.all(np.diff(mergedr[:, : cand.shape[1]], axis=1) <= 0)
825
+ _maxdiff = np.diff(mergedr[:, : cand.shape[1]], axis=1).max()
826
+ assert _maxdiff <= 1e-3, _maxdiff
703
827
  assert np.greater_equal(np.isneginf(mergedl[:, : cand.shape[1]]), cand == -1).all()
704
828
  assert (cand < 0).sum() >= nbye
705
829
  if sorting.labels is not None:
@@ -734,7 +858,7 @@ def _combine_loop(
734
858
  continue
735
859
 
736
860
  eq_ncandj = rcand[j + 1 :] == ncandj
737
- if eq_ncandj.sum() <= 1:
861
+ if eq_ncandj.sum() < 1:
738
862
  continue
739
863
 
740
864
  rsum = mergedr[s, j]
@@ -800,6 +924,8 @@ def deduplicate_spikes(
800
924
  ndrop = 0
801
925
  for unit_id in unit_ids:
802
926
  in_unit = np.flatnonzero(new_labels == unit_id)
927
+ if in_unit.size <= 1:
928
+ continue
803
929
  t = times_samples[in_unit]
804
930
  dt = np.diff(t)
805
931
  if dt.min() > radius_samples:
@@ -870,3 +996,152 @@ def _dedup_unit_loop(
870
996
  break
871
997
 
872
998
  i0 = i1
999
+
1000
+
1001
+ @databag
1002
+ class CoentropyResult:
1003
+ coentropy: np.ndarray
1004
+ """KxK; reduction of entropy per cooccurrence due to merging pair"""
1005
+
1006
+ cooccurrence: np.ndarray
1007
+ """KxK; number of times these units score the same spike"""
1008
+
1009
+ rival_count: np.ndarray
1010
+ """KxK; number of times one unit scores a spike where the other is top"""
1011
+
1012
+ occurrence: np.ndarray
1013
+ """K; number of times the unit appears in the candidates at all"""
1014
+
1015
+ cov: np.ndarray
1016
+ """KxK; rival count / max pair count (rival diag)"""
1017
+
1018
+ iou: np.ndarray
1019
+ """KxK; rival count over pair sum"""
1020
+
1021
+
1022
+ def coentropy_merge_mask(
1023
+ sorting: DARTsortSorting,
1024
+ min_coentropy: float,
1025
+ coverage_threshold: float,
1026
+ iou_threshold: float,
1027
+ gmm_prefix=("merged", "gmm"),
1028
+ ) -> tuple[np.ndarray, CoentropyResult]:
1029
+ """
1030
+ Parameters
1031
+ ----------
1032
+ sorting : DARTsortSorting
1033
+ min_coentropy : float
1034
+ Must be met by pair for mask=True
1035
+ min_coverage : float
1036
+ Pairs such that at least one unit in each pair has
1037
+ rival_count/count > mincov are allowed
1038
+ iou_threshold: float
1039
+ Pairs with rival iou > iouthresh are allowed
1040
+ """
1041
+ c = coentropy(sorting, gmm_prefix=gmm_prefix)
1042
+ assert c is not None
1043
+
1044
+ mask = np.logical_or(c.cov >= coverage_threshold, c.iou >= iou_threshold)
1045
+ mask = np.logical_and(c.coentropy >= min_coentropy, mask)
1046
+ np.fill_diagonal(mask, True)
1047
+ return mask, c
1048
+
1049
+
1050
+ def coentropy(
1051
+ sorting: DARTsortSorting,
1052
+ gmm_prefix=("merged", "gmm"),
1053
+ ) -> CoentropyResult | None:
1054
+ """Calculate entropy reduction due to merging pairs."""
1055
+ for k in gmm_prefix:
1056
+ cands = getattr(sorting, f"{k}_candidates", None)
1057
+ resps = getattr(sorting, f"{k}_responsibilities", None)
1058
+ if cands is not None:
1059
+ assert resps is not None
1060
+ break
1061
+ else:
1062
+ return None
1063
+
1064
+ k = sorting.n_units
1065
+ resps = resps[:, : cands.shape[1]].astype(np.float64)
1066
+ coentropy = np.zeros((k, k))
1067
+ cooccurrence = np.zeros((k, k), dtype=np.int64)
1068
+ rival_count = np.zeros((k, k), dtype=np.int64)
1069
+ occurrence = np.zeros((k,), dtype=np.int64)
1070
+ _calc_coentropy(coentropy, cooccurrence, rival_count, occurrence, cands, resps)
1071
+ rival_count += rival_count.T
1072
+ cdiag = np.diagonal(rival_count)
1073
+ assert (cdiag % 2 == 0).all()
1074
+ np.fill_diagonal(rival_count, cdiag // 2)
1075
+ coentropy += coentropy.T
1076
+ cooccurrence += cooccurrence.T
1077
+
1078
+ # rival count diagonal is just unit top count (not exactly label count,
1079
+ # since it doesn't account for noise assignments)
1080
+ counts = np.diagonal(rival_count)
1081
+ counts = np.maximum(counts, 1)
1082
+
1083
+ cov = rival_count / counts
1084
+ cov = np.minimum(cov, cov.T)
1085
+
1086
+ # this is a disjoint union, since it's the top-label count
1087
+ union = counts[:, None] + counts[None, :]
1088
+ iou = rival_count / union
1089
+
1090
+ return CoentropyResult(
1091
+ coentropy=coentropy,
1092
+ cooccurrence=cooccurrence,
1093
+ rival_count=rival_count,
1094
+ occurrence=occurrence,
1095
+ cov=cov,
1096
+ iou=iou,
1097
+ )
1098
+
1099
+
1100
+ @numba.njit(parallel=True)
1101
+ def _calc_coentropy(
1102
+ coentropy: np.ndarray,
1103
+ cooccurrence: np.ndarray,
1104
+ rival_count: np.ndarray,
1105
+ occurrence: np.ndarray,
1106
+ cands: np.ndarray,
1107
+ resps: np.ndarray,
1108
+ ):
1109
+ for i in numba.prange(cands.shape[0]): # ty: ignore
1110
+ u = cands[i]
1111
+ q = resps[i]
1112
+ log_q = np.log(q)
1113
+ np.nan_to_num(log_q, copy=False, neginf=0.0)
1114
+ dh = q * log_q
1115
+
1116
+ ui0 = u[0]
1117
+ qi0 = q[0]
1118
+ dhi0 = dh[0]
1119
+
1120
+ occurrence[ui0] += 1
1121
+ rival_count[ui0, ui0] += 1
1122
+
1123
+ for j in range(1, cands.shape[1]):
1124
+ uj = u[j]
1125
+ if uj < 0:
1126
+ break
1127
+
1128
+ ii = min(ui0, uj)
1129
+ jj = max(ui0, uj)
1130
+
1131
+ occurrence[uj] += 1
1132
+ rival_count[ui0, uj] += 1
1133
+
1134
+ cij = cooccurrence[ii, jj] + 1
1135
+ cooccurrence[ii, jj] = cij
1136
+
1137
+ # change in entropy due to merging uj, uk:
1138
+ # subtract their current contribution, add the new contribution
1139
+ # we want reduction of entropy, so this is the negative of that!
1140
+ qij = q[j] + qi0
1141
+ dhij = dh[j] + dhi0
1142
+ if qij > 0:
1143
+ dhij -= qij * np.log(qij)
1144
+
1145
+ # Welford mean of -dh
1146
+ cur_coent = coentropy[ii, jj]
1147
+ coentropy[ii, jj] = cur_coent + (-dhij - cur_coent) / cij
@@ -19,7 +19,16 @@ from ..util.job_util import ensure_computation_config
19
19
  from ..util.main_util import ds_save_intermediate_labels
20
20
  from ..util.motion import MotionInfo
21
21
  from ..util.multiprocessing_util import handle_negative_jobs
22
- from . import agglomerate, cluster_util, density, forward_backward, mixture, refine_util
22
+ from ..util.torch_util import cleanup_and_log_gpu_usage
23
+ from . import (
24
+ agglomerate,
25
+ cluster_util,
26
+ density,
27
+ forward_backward,
28
+ kmeans,
29
+ mixture,
30
+ refine_util,
31
+ )
23
32
  from .clustering_features import SimpleMatrixFeatures, StableWaveformFeatures
24
33
 
25
34
  if TYPE_CHECKING:
@@ -292,6 +301,10 @@ class DensityPeaksClusterer(Clusterer):
292
301
  workers=-1,
293
302
  uhdversion=False,
294
303
  random_seed=0,
304
+ kmeans_cleanup=True,
305
+ kmeans_max_sigma: float = 5.0,
306
+ kmeans_iter=100,
307
+ device=None,
295
308
  **kwargs,
296
309
  ):
297
310
  super().__init__(**kwargs)
@@ -308,6 +321,10 @@ class DensityPeaksClusterer(Clusterer):
308
321
  self.workers = workers
309
322
  self.uhdversion = uhdversion
310
323
  self.random_seed = random_seed
324
+ self.kmeans_cleanup = kmeans_cleanup
325
+ self.kmeans_max_sigma = kmeans_max_sigma
326
+ self.kmeans_iter = kmeans_iter
327
+ self.device = device
311
328
 
312
329
  @classmethod
313
330
  def from_config(
@@ -335,6 +352,7 @@ class DensityPeaksClusterer(Clusterer):
335
352
  outlier_radius=clustering_cfg.outlier_radius,
336
353
  outlier_neighbor_count=clustering_cfg.outlier_neighbor_count,
337
354
  workers=workers,
355
+ device=computation_cfg.actual_device(),
338
356
  uhdversion=uhdversion,
339
357
  computation_cfg=computation_cfg,
340
358
  waveform_cfg=waveform_cfg,
@@ -342,6 +360,9 @@ class DensityPeaksClusterer(Clusterer):
342
360
  save_labels_dir=save_labels_dir,
343
361
  labels_fmt=labels_fmt,
344
362
  sampling_cfg=clustering_cfg.sampling_cfg,
363
+ kmeans_cleanup=clustering_cfg.dpc_kmeans_cleanup,
364
+ kmeans_iter=clustering_cfg.kmeans_iter,
365
+ kmeans_max_sigma=clustering_cfg.gmmdpc_max_sigma,
345
366
  )
346
367
 
347
368
  def _cluster(
@@ -425,6 +446,19 @@ class DensityPeaksClusterer(Clusterer):
425
446
  kdtree = None
426
447
  labels = res["labels"]
427
448
 
449
+ if self.kmeans_cleanup:
450
+ kres = kmeans.truncated_kmeans_from_labels(
451
+ X=X,
452
+ labels=labels,
453
+ device=self.device,
454
+ max_sigma=self.kmeans_max_sigma,
455
+ n_iter=self.kmeans_iter,
456
+ )
457
+ assert kres.labels is not None
458
+ labels = kres.labels.numpy(force=True)
459
+ del kres
460
+ cleanup_and_log_gpu_usage(self.computation_cfg, "DPC->kmeans")
461
+
428
462
  labels = cluster_util.decrumb(
429
463
  labels, min_size=self.remove_clusters_smaller_than, in_place=True
430
464
  )
@@ -821,6 +855,7 @@ class AgglomerateRefinement(Refinement):
821
855
  recording: BaseRecording | None,
822
856
  motion: MotionInfo,
823
857
  ):
858
+ assert recording is not None
824
859
  return agglomerate.agglomerate(
825
860
  recording=recording,
826
861
  sorting=sorting,
@@ -25,15 +25,6 @@ from ..util.py_util import databag
25
25
  from ..util.waveform_util import single_channel_index
26
26
  from . import cluster_util
27
27
 
28
- default_clustering_features_cfg = ClusteringFeaturesConfig()
29
- minimal_features_cfg = ClusteringFeaturesConfig(
30
- n_main_channel_pcs=0,
31
- use_amplitude=False,
32
- use_signed_amplitude=False,
33
- use_x=False,
34
- use_z=False,
35
- )
36
-
37
28
  logger = get_logger(__name__)
38
29
 
39
30
 
@@ -77,8 +68,14 @@ class SimpleMatrixFeatures:
77
68
  )
78
69
  if xyza is not None:
79
70
  x = xyza[:, 0]
71
+ if not _allfinite(x):
72
+ raise ValueError(_numbers_error_str("x", x))
80
73
  z = xyza[:, 2]
74
+ if not _allfinite(z):
75
+ raise ValueError(_numbers_error_str("z", z))
81
76
  z_reg = motion.correct_s(t_s, z)
77
+ if not _allfinite(z_reg):
78
+ raise ValueError(_numbers_error_str("z_reg", z_reg))
82
79
  else:
83
80
  x = z = z_reg = None
84
81
 
@@ -99,10 +96,14 @@ class SimpleMatrixFeatures:
99
96
  amp = getattr(sorting, clustering_features_cfg.amplitudes_dataset_name)
100
97
  if clustering_features_cfg.use_amplitude:
101
98
  assert amp is not None
99
+ if not _allfinite(amp):
100
+ raise ValueError(_numbers_error_str("amp", amp))
102
101
  ampft = amp.copy()
103
102
  if clustering_features_cfg.log_transform_amplitude:
104
103
  ampft = np.log(clustering_features_cfg.amp_log_c + ampft)
105
104
  ampft *= clustering_features_cfg.amp_scale
105
+ if not _allfinite(ampft):
106
+ raise ValueError(_numbers_error_str("ampft", ampft))
106
107
  features.append(ampft[:, None])
107
108
 
108
109
  v = getattr(sorting, clustering_features_cfg.voltages_dataset_name, None)
@@ -123,6 +124,8 @@ class SimpleMatrixFeatures:
123
124
  rank=clustering_features_cfg.n_main_channel_pcs,
124
125
  dataset_name=clustering_features_cfg.pca_dataset_name,
125
126
  )
127
+ if not _allfinite(pcs):
128
+ raise ValueError(_numbers_error_str("No motion pcs", pcs))
126
129
  elif do_pcs and clustering_features_cfg.motion_aware:
127
130
  shifts, n_pitches_shift = motion.pitch_shifts(
128
131
  sorting=sorting,
@@ -141,6 +144,8 @@ class SimpleMatrixFeatures:
141
144
  mask = np.broadcast_to(mask, len(schan))
142
145
  if hasattr(sorting, clustering_features_cfg.pca_dataset_name):
143
146
  pcs = getattr(sorting, clustering_features_cfg.pca_dataset_name)
147
+ if not _allfinite(pcs):
148
+ raise ValueError(_numbers_error_str("sorting pcs", pcs))
144
149
  erp, pcs = interpolate_by_chunk(
145
150
  mask=mask,
146
151
  dataset=pcs,
@@ -153,6 +158,8 @@ class SimpleMatrixFeatures:
153
158
  params=clustering_features_cfg.interp_params,
154
159
  )
155
160
  pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
161
+ if not _allfinite(pcs):
162
+ raise ValueError(_numbers_error_str("sorting interp pcs", pcs))
156
163
  else:
157
164
  assert sorting.parent_h5_path is not None
158
165
  with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
@@ -168,20 +175,25 @@ class SimpleMatrixFeatures:
168
175
  params=clustering_features_cfg.interp_params,
169
176
  )
170
177
  pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
178
+ if not _allfinite(pcs):
179
+ raise ValueError(_numbers_error_str("h5 interp pcs", pcs))
171
180
 
172
181
  if do_pcs:
173
182
  assert pcs is not None
174
- if clustering_features_cfg.pc_transform == "log":
183
+ pctf = clustering_features_cfg.pc_transform
184
+ if pctf == "log":
175
185
  pcs = signed_log1p(
176
186
  pcs, pre_scale=clustering_features_cfg.pc_pre_transform_scale
177
187
  )
178
- elif clustering_features_cfg.pc_transform == "sqrt":
188
+ elif pctf == "sqrt":
179
189
  pcs = signed_sqrt_transform(
180
190
  pcs, pre_scale=clustering_features_cfg.pc_pre_transform_scale
181
191
  )
182
192
  else:
183
- assert clustering_features_cfg.pc_transform in ("none", None)
193
+ assert pctf in ("none", None)
184
194
  pcs *= clustering_features_cfg.pc_scale
195
+ if not _allfinite(pcs):
196
+ raise ValueError(_numbers_error_str(f"{pctf} pcs", pcs))
185
197
  if torch.is_tensor(pcs):
186
198
  pcs = pcs.numpy(force=True)
187
199
  features.append(pcs)
@@ -310,3 +322,16 @@ def signed_sqrt_transform(x, pre_scale=1.0):
310
322
  xx.sub_(1.0)
311
323
  xx.mul_(torch.sign(x))
312
324
  return xx
325
+
326
+
327
+ def _allfinite(x):
328
+ if isinstance(x, torch.Tensor):
329
+ return x.isfinite().all()
330
+ else:
331
+ return np.isfinite(x).all()
332
+
333
+
334
+ def _numbers_error_str(name: str, x: np.ndarray):
335
+ if isinstance(x, torch.Tensor):
336
+ x = x.numpy(force=True)
337
+ return f"{name}: {np.isposinf(x).sum()} +inf, {np.isneginf(x).sum()} -inf, {np.isnan(x).sum()} nan."