DARTsort 0.5.11__tar.gz → 0.5.12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. {dartsort-0.5.11 → dartsort-0.5.12}/PKG-INFO +1 -1
  2. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/agglomerate.py +4 -2
  3. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/kmeans.py +1 -7
  4. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/mixture.py +68 -41
  5. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/refine_util.py +38 -141
  6. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/config.py +2 -2
  7. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/analysis.py +1 -0
  8. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/simkit.py +24 -15
  9. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/decollider.py +2 -2
  10. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/reduction.py +0 -1
  11. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/data_util.py +13 -7
  12. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/internal_config.py +1 -1
  13. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/motion.py +10 -1
  14. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/multiprocessing_util.py +1 -1
  15. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/registration_util.py +13 -2
  16. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/analysis_plots.py +1 -1
  17. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/vismain.py +0 -2
  18. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/PKG-INFO +1 -1
  19. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_dartsort.py +3 -2
  20. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_simkit.py +1 -1
  21. {dartsort-0.5.11 → dartsort-0.5.12}/.github/workflows/ci.yml +0 -0
  22. {dartsort-0.5.11 → dartsort-0.5.12}/.github/workflows/ci_windows.yml +0 -0
  23. {dartsort-0.5.11 → dartsort-0.5.12}/.github/workflows/deploy.yml +0 -0
  24. {dartsort-0.5.11 → dartsort-0.5.12}/.github/workflows/docs.yml +0 -0
  25. {dartsort-0.5.11 → dartsort-0.5.12}/.gitignore +0 -0
  26. {dartsort-0.5.11 → dartsort-0.5.12}/README.md +0 -0
  27. {dartsort-0.5.11 → dartsort-0.5.12}/docs/index.md +0 -0
  28. {dartsort-0.5.11 → dartsort-0.5.12}/docs/main_api.md +0 -0
  29. {dartsort-0.5.11 → dartsort-0.5.12}/docs/toolkit_api.md +0 -0
  30. {dartsort-0.5.11 → dartsort-0.5.12}/environment.yml +0 -0
  31. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/.keep +0 -0
  32. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/basic_integration_test.py +0 -0
  33. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/ci diag.py +0 -0
  34. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/dartsort_dredge_demo.ipynb +0 -0
  35. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/dartsort_dredge_demo.py +0 -0
  36. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/data test.py +0 -0
  37. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/dead reckoning.py +0 -0
  38. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/denoise np1.py +0 -0
  39. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/denoise np2.py +0 -0
  40. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/exploratory.py +0 -0
  41. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/geom debug.py +0 -0
  42. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/hdbscan_clustering_5min.py +0 -0
  43. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/hybrid 10.py +0 -0
  44. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/hybrid cluster.py +0 -0
  45. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/hybrid_11.py +0 -0
  46. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/initial training.py +0 -0
  47. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/make hybrid.py +0 -0
  48. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/mgh.py +0 -0
  49. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/np1 feats.py +0 -0
  50. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/np2 processing version c.py +0 -0
  51. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/paperfigs.py +0 -0
  52. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/pdists.py +0 -0
  53. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/postcheck.py +0 -0
  54. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/ptp vae.py +0 -0
  55. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/re clust.py +0 -0
  56. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/reg fig loop.py +0 -0
  57. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/reg_sim_exp.py +0 -0
  58. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/relocating.py +0 -0
  59. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/relocating_all.py +0 -0
  60. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/restats +0 -0
  61. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/sortStinger1.py +0 -0
  62. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/sortStinger3.py +0 -0
  63. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/sortStinger4.py +0 -0
  64. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/sortStinger5.py +0 -0
  65. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/sortStinger6.py +0 -0
  66. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/spike farm.py +0 -0
  67. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/ss_ensemble_sim.py +0 -0
  68. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/ss_ensemble_sim_drift.py +0 -0
  69. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/ss_sim.py +0 -0
  70. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/stan diagnostics.py +0 -0
  71. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/storytelling c.py +0 -0
  72. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/storytelling.py +0 -0
  73. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
  74. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/debug_matching.py +0 -0
  75. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_agreement.py +0 -0
  76. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_amplitude_scaling.py +0 -0
  77. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_batch_optim.py +0 -0
  78. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_ibme.py +0 -0
  79. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_mm_initialization.py +0 -0
  80. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_normxcorr1d.py +0 -0
  81. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_reassignment.py +0 -0
  82. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_relocation.py +0 -0
  83. {dartsort-0.5.11 → dartsort-0.5.12}/notebook/thomas.py +0 -0
  84. {dartsort-0.5.11 → dartsort-0.5.12}/pyproject.toml +0 -0
  85. {dartsort-0.5.11 → dartsort-0.5.12}/setup.cfg +0 -0
  86. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/__init__.py +2 -2
  87. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/cli.py +0 -0
  88. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/__init__.py +0 -0
  89. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/cluster_util.py +0 -0
  90. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/clustering.py +0 -0
  91. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/clustering_features.py +0 -0
  92. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/density.py +0 -0
  93. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/forward_backward.py +0 -0
  94. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/merge.py +0 -0
  95. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/modes.py +0 -0
  96. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/detect/__init__.py +0 -0
  97. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/detect/detect.py +0 -0
  98. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/detect/detect_filters.py +0 -0
  99. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/__init__.py +0 -0
  100. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/comparison.py +0 -0
  101. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/hybrid_util.py +0 -0
  102. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  103. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  104. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/simlib.py +0 -0
  105. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/localize/__init__.py +0 -0
  106. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/localize/localize_torch.py +0 -0
  107. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/localize/localize_util.py +0 -0
  108. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/main.py +0 -0
  109. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/__init__.py +0 -0
  110. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/grab.py +0 -0
  111. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching.py +0 -0
  112. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/__init__.py +0 -0
  113. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  114. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/drifty.py +0 -0
  115. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  116. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  117. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  118. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/peel_base.py +0 -0
  119. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/peel_lib.py +0 -0
  120. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/reduction_template.py +0 -0
  121. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/shaver.py +0 -0
  122. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/subtract.py +0 -0
  123. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/threshold.py +0 -0
  124. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  125. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/detect_np1.pt +0 -0
  126. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/detect_np2.pt +0 -0
  127. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  128. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  129. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  130. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  131. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  132. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  133. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/__init__.py +0 -0
  134. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/get_templates.py +0 -0
  135. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/postprocess_util.py +0 -0
  136. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/realignment.py +0 -0
  137. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/template_util.py +0 -0
  138. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/templates.py +0 -0
  139. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/templib.py +0 -0
  140. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/__init__.py +0 -0
  141. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  142. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/all_transformers.py +0 -0
  143. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/amortized_localization.py +0 -0
  144. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/amplitudes.py +0 -0
  145. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/denoising_scorer.py +0 -0
  146. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/enforce_decrease.py +0 -0
  147. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/fixed_prop.py +0 -0
  148. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/interp.py +0 -0
  149. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/localize.py +0 -0
  150. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/matching_denoiser.py +0 -0
  151. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/mixture_classifier.py +0 -0
  152. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/pipeline.py +0 -0
  153. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  154. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/supervised_denoiser.py +0 -0
  155. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/temporal_pca.py +0 -0
  156. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/transform_base.py +0 -0
  157. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/whiten.py +0 -0
  158. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/__init__.py +0 -0
  159. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/cli_util.py +0 -0
  160. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/drift_util.py +0 -0
  161. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/interpolation_util.py +0 -0
  162. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/job_util.py +0 -0
  163. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/logging_util.py +0 -0
  164. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/main_util.py +0 -0
  165. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/more_operators.py +0 -0
  166. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/nn_util.py +0 -0
  167. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/noise_util.py +0 -0
  168. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/peel_util.py +0 -0
  169. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/preprocess_util.py +0 -0
  170. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/py_util.py +0 -0
  171. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/residual_util.py +0 -0
  172. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/sparse_util.py +0 -0
  173. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/spikeio.py +0 -0
  174. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/spiketorch.py +0 -0
  175. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/testing_util/__init__.py +0 -0
  176. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  177. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  178. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/torch_optimization_util.py +0 -0
  179. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/torch_util.py +0 -0
  180. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/waveform_util.py +0 -0
  181. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/__init__.py +0 -0
  182. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/colors.py +0 -0
  183. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/gt.py +0 -0
  184. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/layout.py +0 -0
  185. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/metrics_util.py +0 -0
  186. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/mixture.py +0 -0
  187. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/recanim.py +0 -0
  188. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/scatterplots.py +0 -0
  189. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/sorting.py +0 -0
  190. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/unit.py +0 -0
  191. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/unit_comparison.py +0 -0
  192. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/versus.py +0 -0
  193. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/waveforms.py +0 -0
  194. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/SOURCES.txt +0 -0
  195. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/dependency_links.txt +0 -0
  196. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/entry_points.txt +0 -0
  197. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/requires.txt +0 -0
  198. {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/top_level.txt +0 -0
  199. {dartsort-0.5.11 → dartsort-0.5.12}/tests/conftest.py +0 -0
  200. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_alignment.py +0 -0
  201. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_cluster_util.py +0 -0
  202. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_clustering.py +0 -0
  203. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_config.py +0 -0
  204. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_data_util.py +0 -0
  205. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_density.py +0 -0
  206. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_detect.py +0 -0
  207. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_drift_util.py +0 -0
  208. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_drifty_matching.py +0 -0
  209. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_enforce_decrease.py +0 -0
  210. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_grab_and_featurize.py +0 -0
  211. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_kmeans.py +0 -0
  212. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_linalg.py +0 -0
  213. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_localize_torch.py +0 -0
  214. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_matching.py +0 -0
  215. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_mixture_classifier.py +0 -0
  216. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_modeling.py +0 -0
  217. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_spiketorch.py +0 -0
  218. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_subtract.py +0 -0
  219. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_templates.py +0 -0
  220. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_threshold.py +0 -0
  221. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_torch_optimization_util.py +0 -0
  222. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_transform.py +0 -0
  223. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_util.py +0 -0
  224. {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_waveform_util.py +0 -0
  225. {dartsort-0.5.11 → dartsort-0.5.12}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.11
3
+ Version: 0.5.12
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -211,7 +211,7 @@ def agglomerate(
211
211
  agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
212
212
 
213
213
  agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
214
- new_ids = reorder[new_ids]
214
+ new_ids = reorder[np.unique(new_ids, return_inverse=True)[1]]
215
215
 
216
216
  return Agglomeration(
217
217
  agglomerated_sorting=agg_sorting,
@@ -957,7 +957,9 @@ def _dedup_unit_loop(
957
957
  score_slice = scores[i0 : i1 + 1]
958
958
  dt_slice = dt[i0:i1]
959
959
  order = np.argsort(score_slice)
960
- for oo in order:
960
+ # putting the -1 there ensures at least one spike is kept
961
+ # that case is only relevant when 0 in dt_slice
962
+ for oo in order[:-1]:
961
963
  # discard spike i0 + oo
962
964
  ii = i0 + oo
963
965
  discard[ii] = True
@@ -24,7 +24,6 @@ from ..util.sparse_util import (
24
24
  sparse_centroid_distsq,
25
25
  )
26
26
  from ..util.spiketorch import spawn_torch_rg, sqeuc_cdist_known_norm
27
- from ..util.torch_util import torch_compile, torch_compiler
28
27
 
29
28
  logger = get_logger(__name__)
30
29
 
@@ -78,6 +77,7 @@ def kmeanspp(
78
77
 
79
78
  simple_case = (not min_distance) and (not skip_assignment)
80
79
  if simple_case:
80
+ assert assignments is not None
81
81
  _kmeanspp_simple_loop(
82
82
  X=X,
83
83
  Xnormsq=Xnormsq,
@@ -475,19 +475,16 @@ def kmeans(
475
475
  )
476
476
 
477
477
 
478
- @torch.compiler.disable
479
478
  def _uniform(gen: torch.Generator, buf: Tensor):
480
479
  return buf.uniform_(generator=gen)
481
480
 
482
481
 
483
- @torch_compiler(fullgraph=False)
484
482
  def _one_gumbel_nolog(p: Tensor, gen: torch.Generator, buf: Tensor):
485
483
  z = _uniform(gen, buf)
486
484
  z = z.log_()
487
485
  return torch.divide(p, z, out=z).argmin(dim=0)
488
486
 
489
487
 
490
- @torch_compile
491
488
  def _sqeuc(X: Tensor, Xnormsq: Tensor, Y: Tensor, out: Tensor):
492
489
  torch.addmm(Xnormsq, X, Y.transpose(0, 1), alpha=-2.0, out=out)
493
490
  Ynormsq = (Y**2).sum(dim=1)
@@ -495,7 +492,6 @@ def _sqeuc(X: Tensor, Xnormsq: Tensor, Y: Tensor, out: Tensor):
495
492
  return out
496
493
 
497
494
 
498
- @torch_compiler(fullgraph=False)
499
495
  def _kmeanspp_simple_loop(
500
496
  *,
501
497
  X: Tensor,
@@ -531,7 +527,6 @@ def _kmeanspp_simple_loop(
531
527
  torch.minimum(dists, newdists, out=dists)
532
528
 
533
529
 
534
- @torch_compiler(fullgraph=False)
535
530
  def _kmeanspp_noassign_loop(
536
531
  *,
537
532
  X: Tensor,
@@ -563,7 +558,6 @@ def _kmeanspp_noassign_loop(
563
558
  torch.minimum(dists, newdists, out=dists)
564
559
 
565
560
 
566
- @torch_compiler(fullgraph=False)
567
561
  def _kmeans_main_loop(
568
562
  n_iter: int,
569
563
  e: Tensor,
@@ -107,8 +107,8 @@ if TYPE_CHECKING:
107
107
 
108
108
  logger = get_logger(__name__)
109
109
  pnoid = logger.isEnabledFor(DARTSORTVERBOSE)
110
- prop_check_atol = 2e-4
111
- prop_check_rtol = 1e-6
110
+ prop_check_atol = 1e-3
111
+ prop_check_rtol = 1e-5
112
112
 
113
113
 
114
114
  # -- main
@@ -1049,7 +1049,7 @@ class DenseSpikeData:
1049
1049
  """
1050
1050
  unit_ids = torch.as_tensor(unit_ids, device=self.neighborhood_ids.device)
1051
1051
  covered = self.lut_coverage(unit_ids, lut)
1052
- candidates = torch.where(covered, unit_ids[None, :], -1)
1052
+ candidates = torch.where(covered, unit_ids[None, :], -1).long()
1053
1053
  candidate_counts = covered.sum(dim=1).cpu()
1054
1054
 
1055
1055
  batches = []
@@ -1501,7 +1501,7 @@ class StreamingSpikeData(BatchedSpikeData):
1501
1501
  assert self.max_n_total is not None
1502
1502
  assert self.n_total == self.max_n_total
1503
1503
  proposals = full_proposal_by_neighb(self.un_adj_lut, self.n_total)
1504
- self.proposals = proposals.to(self.device)
1504
+ self.proposals = proposals.to(self.device, dtype=torch.long)
1505
1505
 
1506
1506
  def _update_sizes_from_n_units(self, n_units: int):
1507
1507
  if self.max_n_total is None or self.max_n_explore is None:
@@ -1657,6 +1657,7 @@ class TruncatedSpikeData(BatchedSpikeData):
1657
1657
  explore_neighb_steps: int = 1,
1658
1658
  neighb_overlap: float = 0.75,
1659
1659
  batch_size: int = 128,
1660
+ allow_gaps=False,
1660
1661
  ) -> Self:
1661
1662
  assert len(x) == len(neighborhoods.b.neighborhood_ids)
1662
1663
  xt, whitenedx, CmoCooinvx, noise_logliks = _whiten_impute_and_noise_score(
@@ -1685,9 +1686,11 @@ class TruncatedSpikeData(BatchedSpikeData):
1685
1686
  )
1686
1687
  assert self.candidates is not None
1687
1688
  self.candidates[:, 0] = labels
1688
- if pnoid:
1689
- _k = labels.amax() + 1 + int((labels < 0).any())
1690
- assert labels.unique().shape == (_k,)
1689
+ if pnoid and not allow_gaps:
1690
+ luniq = labels.unique().cpu()
1691
+ luniq = luniq[luniq >= 0]
1692
+ assert luniq.numel() > 0, str(luniq.tolist())
1693
+ assert luniq.numel() == luniq.amax() + 1, str(luniq.tolist())
1691
1694
  return self
1692
1695
 
1693
1696
  def update(
@@ -1708,7 +1711,7 @@ class TruncatedSpikeData(BatchedSpikeData):
1708
1711
  self.un_adj_lut.b.lut,
1709
1712
  (0, 0, 0, 1),
1710
1713
  value=self.un_adj_lut.b.unit_ids.shape[0],
1711
- )
1714
+ ).long()
1712
1715
  top_lut_ixs = lut_padded[
1713
1716
  self.candidates[:, : self.n_candidates], self.neighborhood_ids[:, None]
1714
1717
  ]
@@ -3834,7 +3837,7 @@ class TruncatedMixtureModel(BaseMixtureModel):
3834
3837
  assert n_new_units == sum(lp.numel() for lp in new_log_props)
3835
3838
  if self.signal_rank:
3836
3839
  assert all(b is not None for b in new_bases)
3837
- assert n_new_units == sum(b.shape[0] for b in new_bases) # type: ignore
3840
+ assert n_new_units == sum(b.shape[0] for b in new_bases) # type: ignore # ty: ignore[x]
3838
3841
 
3839
3842
  # resize params to allow space for the new guys
3840
3843
  Korig = self.n_units
@@ -4243,6 +4246,7 @@ def get_truncated_datasets(
4243
4246
  neighb_cov=neighb_cov,
4244
4247
  seed=rg,
4245
4248
  batch_size=refinement_cfg.eval_batch_size,
4249
+ allow_gaps=True,
4246
4250
  )
4247
4251
  assert torch.equal(
4248
4252
  val_data.neighborhoods.b.neighborhoods,
@@ -4427,6 +4431,8 @@ def instantiate_and_bootstrap_tmm(
4427
4431
  sorting, min_spikes=refinement_cfg.min_count
4428
4432
  )
4429
4433
  sorting = sorting.flatten()
4434
+ if not sorting.n_units:
4435
+ raise ValueError("No units to cluster.")
4430
4436
 
4431
4437
  neighb_cov, erp, train_data, val_data, full_data, noise, train_ixs, val_ixs = (
4432
4438
  get_truncated_datasets(
@@ -5554,7 +5560,10 @@ def _evaluate_single_demolition(
5554
5560
 
5555
5561
  # evaluate effect on heldout set
5556
5562
  eval_scores_adj = proportion_adjust_scores(
5557
- scores=eval_scores, orig_log_props=orig_log_props, new_log_props=new_log_props
5563
+ scores=eval_scores,
5564
+ orig_log_props=orig_log_props,
5565
+ new_log_props=new_log_props,
5566
+ sort=False,
5558
5567
  )
5559
5568
  return ecl(
5560
5569
  resps=eval_scores_adj.responsibilities,
@@ -6070,7 +6079,7 @@ def combine_luts(*luts: NeighborhoodLUT) -> NeighborhoodLUT:
6070
6079
  return luts[0]
6071
6080
  adj = luts[0].b.lut < luts[0].b.unit_ids.shape[0]
6072
6081
  for ll in luts[1:]:
6073
- adj.logical_or_(ll.lut < ll.unit_ids.shape[0]) # type: ignore
6082
+ adj.logical_or_(ll.lut < ll.unit_ids.shape[0]) # type: ignore # ty: ignore[x]
6074
6083
  unit_ids, neighb_ids = adj.nonzero(as_tuple=True)
6075
6084
  lut = torch.full_like(luts[0].b.lut, unit_ids.shape[0])
6076
6085
  lut[unit_ids, neighb_ids] = torch.arange(unit_ids.shape[0], device=lut.device)
@@ -6106,7 +6115,7 @@ def candidate_search_sets(
6106
6115
  tops, topunits = torch.topk(s, k=n_search, dim=1)
6107
6116
  topunits.masked_fill_(tops == 0, -1)
6108
6117
  # pad with row of -1s for the invalid lut ixs (===n_lut)
6109
- topunits = F.pad(topunits, (0, 0, 0, 1), value=-1)
6118
+ topunits = F.pad(topunits, (0, 0, 0, 1), value=-1).long()
6110
6119
 
6111
6120
  return topunits
6112
6121
 
@@ -6120,7 +6129,7 @@ def full_proposal_by_neighb(lut: NeighborhoodLUT, max_proposed: int):
6120
6129
  """Returns n_neighbs x max possible proposed units array, basically transposing the LUT's nonzeros."""
6121
6130
  n_neighbs = lut.b.lut.shape[1]
6122
6131
  n_lut = lut.b.unit_ids.shape[0]
6123
- proposals = torch.full((n_neighbs, max_proposed), -1)
6132
+ proposals = torch.full((n_neighbs, max_proposed), -1, dtype=torch.long)
6124
6133
  for neighb_id in range(n_neighbs):
6125
6134
  row = lut.b.lut[:, neighb_id]
6126
6135
  (row_valid,) = (row < n_lut).nonzero(as_tuple=True)
@@ -6157,13 +6166,13 @@ def _fill_blank_labels(
6157
6166
 
6158
6167
  # need full coverage of neighborhoods here. would prefer only to branch out with
6159
6168
  # explore as needed, so make those probs tiny.
6160
- keep_same_adj = un_adj.sum(0).min().cpu().item() > 0
6169
+ keep_same_adj = un_adj.sum(0).amin().cpu().item() > 0
6161
6170
  n_new_units = 0
6162
6171
  if keep_same_adj:
6163
6172
  adj = un_adj
6164
6173
  else:
6165
6174
  adj = un_adj + explore_adj * torch.finfo(explore_adj.dtype).tiny
6166
- uncovered = adj.sum(0).min().cpu().item() == 0
6175
+ uncovered = adj.sum(0).amin().cpu().item() == 0
6167
6176
  if uncovered:
6168
6177
  uncovered_adj = adj.clone()
6169
6178
  else:
@@ -6176,7 +6185,7 @@ def _fill_blank_labels(
6176
6185
  if adj.sum(0).min().cpu().item() > 0:
6177
6186
  break
6178
6187
 
6179
- still_uncovered = adj.sum(0).min().cpu().item() == 0
6188
+ still_uncovered = adj.sum(0).amin().cpu().item() == 0
6180
6189
  if still_uncovered and ensure_coverage_only and allow_new_units:
6181
6190
  # make new units by getting connected components of the uncovered
6182
6191
  # neighborhood graph
@@ -6221,13 +6230,20 @@ def _fill_blank_labels(
6221
6230
  if ensure_coverage_only:
6222
6231
  return None, n_new_units
6223
6232
 
6233
+ if pnoid:
6234
+ assert adj.amin() >= 0
6235
+
6224
6236
  for i0 in range(0, Nblank, batch_size):
6225
6237
  i1 = min(Nblank, i0 + batch_size)
6226
6238
  ii = blank[i0:i1]
6227
6239
  nn = neighborhood_ids[ii]
6228
- p = adj[:, nn].mT
6240
+ p = adj.T[nn]
6241
+ p_is_0 = p.sum(1) <= 0.0
6242
+ p[:, 0].clamp_min_(p_is_0.to(p))
6229
6243
  draws = torch.multinomial(p, 1, replacement=True, generator=gen)
6230
- labels[ii] = draws.view(ii.shape[0])
6244
+ draws = draws.view(ii.shape[0])
6245
+ draws.masked_fill_(p_is_0, -1)
6246
+ labels[ii] = draws
6231
6247
 
6232
6248
  return keep_same_adj, n_new_units
6233
6249
 
@@ -6306,8 +6322,10 @@ def _get_explore_sampling_data(
6306
6322
  p.masked_fill_(pzero, torch.finfo(p.dtype).tiny)
6307
6323
 
6308
6324
  # pad p with an extra LUT index (row), inds with a -1
6309
- p = F.pad(p, (0, 0, 0, 1))
6310
- inds = F.pad(inds, (0, 0, 0, 1), value=-1)
6325
+ # the tiny mops up cases with no overlaps, and that case is
6326
+ # sometimes allowed by the logic which gets us here
6327
+ p = F.pad(p, (0, 0, 0, 1), value=torch.finfo(p.dtype).tiny)
6328
+ inds = F.pad(inds, (0, 0, 0, 1), value=-1).long()
6311
6329
 
6312
6330
  return p, inds
6313
6331
 
@@ -6350,8 +6368,10 @@ def _dedup_candidates(candidates):
6350
6368
 
6351
6369
  def _count_candidates(candidates, batch_candidate_counts, batch_size):
6352
6370
  counts = candidates.new_zeros(batch_candidate_counts.shape)
6353
- for b, i0 in enumerate(range(0, candidates.shape[0], batch_size)):
6354
- counts[b] = (candidates[i0 : i0 + batch_size] >= 0).sum()
6371
+ batch_iter = range(0, candidates.shape[0], batch_size)
6372
+ assert counts.shape[0] == len(batch_iter)
6373
+ for b, i0 in enumerate(batch_iter):
6374
+ counts[b] = (candidates[i0 : i0 + batch_size] >= 0).count_nonzero()
6355
6375
  batch_candidate_counts.copy_(counts.cpu())
6356
6376
 
6357
6377
 
@@ -6395,38 +6415,47 @@ def concatenate_scores(scoress: list[Scores], dim=0) -> Scores:
6395
6415
  )
6396
6416
 
6397
6417
 
6398
- def remove_units_from_scores(scores: Scores, unit_ids: Tensor) -> Scores:
6418
+ def remove_units_from_scores(scores: Scores, unit_ids: Tensor, sort=True) -> Scores:
6399
6419
  """Return copy of scores where log_liks for candidates in unit_ids are -inf (and resps are 0)."""
6400
6420
  bye = torch.isin(scores.candidates, unit_ids.to(scores.candidates))
6401
6421
  new_cand = scores.candidates.masked_fill(bye, -1)
6402
6422
  new_log_lik = scores.log_liks.clone()
6403
6423
  new_log_lik[:, : new_cand.shape[1]].masked_fill_(bye, -torch.inf)
6404
- new_resp = new_log_lik.softmax(dim=1)
6405
- return Scores(
6424
+ new_resp = new_log_lik.softmax(dim=1).nan_to_num_()
6425
+ scores = Scores(
6406
6426
  candidates=new_cand,
6407
6427
  log_liks=new_log_lik,
6408
6428
  responsibilities=new_resp,
6409
6429
  duties=scores.duties,
6410
6430
  )
6431
+ if sort:
6432
+ return ensure_sorted_scores(scores)
6433
+ else:
6434
+ return scores
6411
6435
 
6412
6436
 
6413
6437
  def proportion_adjust_scores(
6414
- scores: Scores, orig_log_props: Tensor, new_log_props: Tensor
6438
+ scores: Scores, orig_log_props: Tensor, new_log_props: Tensor, sort=True
6415
6439
  ) -> Scores:
6416
6440
  """What would the scores be if the log props were these, not those?"""
6417
6441
  diff = new_log_props - orig_log_props
6442
+ diff.nan_to_num_()
6418
6443
  diff = F.pad(diff, (0, 1))
6419
6444
  diff = diff[scores.candidates]
6420
6445
  new_log_liks = scores.log_liks.clone()
6421
6446
  if scores.duties is not None:
6422
6447
  diff *= scores.duties[:, None]
6423
6448
  new_log_liks[:, : diff.shape[1]] += diff
6424
- return Scores(
6449
+ scores = Scores(
6425
6450
  log_liks=new_log_liks,
6426
6451
  candidates=scores.candidates,
6427
- responsibilities=new_log_liks.softmax(dim=1),
6452
+ responsibilities=new_log_liks.softmax(dim=1).nan_to_num_(),
6428
6453
  duties=scores.duties,
6429
6454
  )
6455
+ if sort:
6456
+ return ensure_sorted_scores(scores)
6457
+ else:
6458
+ return scores
6430
6459
 
6431
6460
 
6432
6461
  def ensure_sorted_scores(scores: Scores) -> Scores:
@@ -6443,7 +6472,7 @@ def ensure_sorted_scores(scores: Scores) -> Scores:
6443
6472
  assert scores.log_liks.shape[1] == C + 1
6444
6473
  log_liks[:, -1] = scores.log_liks[:, -1]
6445
6474
  if scores.responsibilities is not None:
6446
- responsibilities = F.softmax(log_liks, dim=1)
6475
+ responsibilities = F.softmax(log_liks, dim=1).nan_to_num_()
6447
6476
  else:
6448
6477
  responsibilities = None
6449
6478
 
@@ -6472,18 +6501,14 @@ def drop_units_and_update_scores(
6472
6501
  new_prop *= model_prop / new_prop.sum()
6473
6502
  orig_log_prop = orig_prop.log()
6474
6503
  new_log_prop = new_prop.log()
6475
- updated = torch.logical_not(torch.isclose(orig_log_prop, new_log_prop))
6476
- updated[remove_ids] = False
6477
6504
 
6478
6505
  # update the train scores
6479
6506
  train_scores = proportion_adjust_scores(train_scores, orig_log_prop, new_log_prop)
6480
- train_scores = ensure_sorted_scores(train_scores)
6481
6507
 
6482
6508
  # update other scores
6483
6509
  if scores is not None:
6484
- scores = remove_units_from_scores(scores, remove_ids)
6510
+ scores = remove_units_from_scores(scores, remove_ids, sort=False)
6485
6511
  scores = proportion_adjust_scores(scores, orig_log_prop, new_log_prop)
6486
- scores = ensure_sorted_scores(scores)
6487
6512
 
6488
6513
  return train_scores, scores
6489
6514
 
@@ -6511,7 +6536,6 @@ def mean_responsibilities(
6511
6536
  cand = scores.candidates
6512
6537
  else:
6513
6538
  assert False
6514
-
6515
6539
  assert resp is not None
6516
6540
  assert cand is not None
6517
6541
  ncand = cand.shape[1]
@@ -6525,7 +6549,7 @@ def mean_responsibilities(
6525
6549
  assert n_units > 0
6526
6550
 
6527
6551
  # count candidates per batch
6528
- ncand = (cand >= 0).sum(1)
6552
+ ncand = (cand >= 0).count_nonzero(dim=1)
6529
6553
  padlen = batch_size * int(math.ceil(cand.shape[0] / batch_size))
6530
6554
  if padlen > ncand.shape[0]:
6531
6555
  ncand = F.pad(ncand, (0, padlen - ncand.shape[0]))
@@ -6538,14 +6562,16 @@ def mean_responsibilities(
6538
6562
  for bix, i0 in enumerate(range(0, resp.shape[0], batch_size)):
6539
6563
  i1 = min(resp.shape[0], i0 + batch_size)
6540
6564
  nbatch = i1 - i0
6565
+ if not nbatch:
6566
+ continue
6541
6567
  rsum_batch.zero_()
6542
6568
 
6543
6569
  nc = int(ncand[bix].item())
6544
6570
  cii, cjj = _nonzero_static(cand[i0:i1] >= 0, size=nc).T
6545
-
6546
- c = cand[i0:i1][cii, cjj]
6547
- r = resp[i0:i1][cii, cjj].double()
6548
- rsum_batch.scatter_add_(dim=0, index=c, src=r)
6571
+ if cii.numel():
6572
+ c = cand[i0:i1][cii, cjj].long()
6573
+ r = resp[i0:i1][cii, cjj].double()
6574
+ rsum_batch.scatter_add_(dim=0, index=c, src=r)
6549
6575
  if includes_noise:
6550
6576
  rsum_batch[n_units] += resp[i0:i1, -1].double().sum()
6551
6577
 
@@ -6554,7 +6580,8 @@ def mean_responsibilities(
6554
6580
 
6555
6581
  # check that things didn't explode
6556
6582
  assert resp_mean.isfinite().all(), "Responsibility mean not finite"
6557
- assert torch.all(resp_mean >= 0), "Responsiblity mean negative"
6583
+ assert torch.all(resp_mean >= -prop_check_atol), "Responsiblity mean negative"
6584
+ resp_mean.relu_()
6558
6585
  assert torch.all(resp_mean <= 1.0 + prop_check_atol), (
6559
6586
  f"Responsiblity mean > 1, largest: {resp_mean.amax()}"
6560
6587
  )
@@ -271,35 +271,39 @@ def collision_cleaning_error_filter(
271
271
  # discard bad units
272
272
  keep_mask = np.ones(nu0, dtype=np.bool)
273
273
  bad_ids = np.flatnonzero(dist > refinement_cfg.collision_cleaning_error_threshold)
274
+ logger.dartsortdebug(
275
+ f"Collision-cleaning error filter drops {bad_ids.size} / {nu0} units."
276
+ )
274
277
  keep_mask[bad_ids] = False
275
278
  spike_keep_mask = keep_mask[sorting.labels]
276
279
  new_labels = np.where(spike_keep_mask, sorting.labels, -1)
277
280
 
278
281
  new_props = dict(labels=new_labels)
279
- try:
280
- scores = get_gmm_scores(sorting, prefixes=["gmm"])
281
- scores, _ = drop_units_and_update_scores(
282
- train_scores=scores,
283
- scores=None,
284
- n_units=nu0,
285
- remove_ids=torch.tensor(bad_ids),
286
- )
282
+ if bad_ids.size:
283
+ try:
284
+ scores = get_gmm_scores(sorting, prefixes=["gmm"])
285
+ scores, _ = drop_units_and_update_scores(
286
+ train_scores=scores,
287
+ scores=None,
288
+ n_units=nu0,
289
+ remove_ids=torch.tensor(bad_ids, dtype=torch.long),
290
+ )
287
291
 
288
- # but also fully delete those spikes.
289
- cand = scores.candidates.numpy(force=True)
290
- ll = scores.log_liks.numpy(force=True)
291
- del_mask = np.logical_not(spike_keep_mask)
292
- cand[del_mask] = -1
293
- ll[del_mask, : cand.shape[1]] = -np.inf
294
- new_props["gmm_candidates"] = cand
295
- new_props["gmm_log_liks"] = ll
296
- if scores.responsibilities is not None:
297
- resp = scores.responsibilities.numpy(force=True)
298
- resp[del_mask, : cand.shape[1]] = 0.0
299
- resp[del_mask, -1] = 1.0
300
- new_props["gmm_responsibilities"] = resp
301
- except AttributeError:
302
- pass
292
+ # but also fully delete those spikes.
293
+ cand = scores.candidates.numpy(force=True)
294
+ ll = scores.log_liks.numpy(force=True)
295
+ del_mask = np.logical_not(spike_keep_mask)
296
+ cand[del_mask] = -1
297
+ ll[del_mask, : cand.shape[1]] = -np.inf
298
+ new_props["gmm_candidates"] = cand
299
+ new_props["gmm_log_liks"] = ll
300
+ if scores.responsibilities is not None:
301
+ resp = scores.responsibilities.numpy(force=True)
302
+ resp[del_mask, : cand.shape[1]] = 0.0
303
+ resp[del_mask, -1] = 1.0
304
+ new_props["gmm_responsibilities"] = resp
305
+ except AttributeError:
306
+ pass
303
307
 
304
308
  sorting = sorting.ephemeral_replace(**new_props)
305
309
  sorting = sorting.flatten(include_gmm_properties=True)
@@ -374,6 +378,7 @@ def gmm_isolation_filter(
374
378
  if ngood == gi.isolation.shape[0]:
375
379
  break
376
380
 
381
+ # TODO: why is bad_guy coming through in sorted order?
377
382
  bad_guy = np.argmax(np.where(good, -np.inf, gi.isolation))
378
383
  removed_ids.append(bad_guy)
379
384
 
@@ -384,11 +389,17 @@ def gmm_isolation_filter(
384
389
  update[needs_update[needs_update >= 0]] = True
385
390
  update[good] = False
386
391
 
392
+ logger.dartsortverbose(
393
+ f"Drop {bad_guy} (%s / %s total, %s good).",
394
+ len(removed_ids),
395
+ update.size,
396
+ ngood,
397
+ )
387
398
  scores, _ = drop_units_and_update_scores(
388
399
  train_scores=scores,
389
400
  scores=None,
390
401
  n_units=unit_ids.shape[0],
391
- remove_ids=torch.tensor([bad_guy]),
402
+ remove_ids=torch.tensor([bad_guy], dtype=torch.long),
392
403
  )
393
404
 
394
405
  keep_mask = np.ones(unit_ids.max() + 1)
@@ -565,6 +576,8 @@ def _iso_job(unit_id) -> tuple[float, np.ndarray | None, np.ndarray | None]:
565
576
  try:
566
577
  # select bandwidth based on positive part
567
578
  bw = improved_sheather_jones(lr[:, None])
579
+ if not np.isfinite(bw):
580
+ return np.nan, None, None
568
581
  # reweight near 0
569
582
  weights = norm.sf(0, loc=lr, scale=bw)
570
583
  # reflect and stack
@@ -591,8 +604,7 @@ def _iso_job(unit_id) -> tuple[float, np.ndarray | None, np.ndarray | None]:
591
604
 
592
605
  # find the HIGHEST local max right of 0, if any
593
606
  peak_ix_right = _find_right_peak_index(ev, 1)
594
- evpeak = ev[peak_ix_right]
595
- no_right_peak = peak_ix_right == ev.shape[0] or ev0 > evpeak
607
+ no_right_peak = peak_ix_right == ev.shape[0] or ev0 > ev[peak_ix_right]
596
608
 
597
609
  if no_right_peak:
598
610
  # no isolation
@@ -621,121 +633,6 @@ def _iso_job(unit_id) -> tuple[float, np.ndarray | None, np.ndarray | None]:
621
633
  return iso, domain, kde
622
634
 
623
635
 
624
- # def _iso_job(unit_id) -> tuple[float, np.ndarray | None, np.ndarray | None]:
625
- # p = cast(_GMMIsolationContext, _iso_ctx.ctx)
626
- # assert p is not None
627
-
628
- # # top indices
629
- # (inu,) = (p.cand[:, 0] == unit_id).nonzero(as_tuple=True)
630
- # if inu.numel() <= p.min_count:
631
- # return np.nan, None, None
632
-
633
- # # second best indices
634
- # (byu,) = (p.cand[:, 1] == unit_id).nonzero(as_tuple=True)
635
-
636
- # # likelihood ratio when unit comes first vs second
637
- # in_ll = p.log_liks[inu, :-1]
638
- # # in_denom = in_ll[:, 1:].logsumexp(dim=1)
639
- # in_denom = in_ll[:, 1]
640
- # min_denom, max_denom = in_denom.aminmax()
641
- # in_lr = in_ll[:, 0] - in_denom
642
- # if torch.isneginf(max_denom):
643
- # # this unit is super isolated.
644
- # return 0.0, None, None
645
- # if torch.isneginf(min_denom):
646
- # # first check that enough are finite
647
- # (finite,) = torch.isfinite(in_lr).nonzero(as_tuple=True)
648
- # nfinite = finite.numel()
649
- # if nfinite < p.min_count:
650
- # return np.nan, None, None
651
- # if nfinite / inu.size < p.neighbor_fraction:
652
- # return np.nan, None, None
653
-
654
- # # handle +inf lrs by replacing with neg log liks
655
- # in_lr.clamp_(max=in_ll[:, 0].abs_())
656
-
657
- # # by_lr is finite by construction unless numerical catastrophe happens
658
- # by_lr = p.log_liks[byu, :2].diff(dim=1)[:, 0]
659
- # by_lr.clamp_(min=-1000.0)
660
-
661
- # in_lr = in_lr.double()
662
- # lr = torch.concatenate([by_lr, in_lr])
663
-
664
- # # # weights to avoid being overwhelmed by the "by" category
665
- # # nin = in_lr.numel()
666
- # # nby = by_lr.numel()
667
- # # if nby > nin:
668
- # # weights = torch.empty_like(lr, device="cpu").numpy(force=True)
669
- # # weights[:nby] = nin / nby
670
- # # weights[nby:] = 1.0
671
- # # else:
672
- # # weights = None
673
- # weights = None
674
-
675
- # # fit kde. will symmetrize around 0 to avoid boundary issues (since lr>=0)
676
- # amax = lr.abs().amax().item()
677
- # lr = lr.numpy(force=True)
678
- # try:
679
- # # select bandwidth based on positive part
680
- # bw = improved_sheather_jones(in_lr.numpy(force=True)[:, None])
681
- # kde = FFTKDE(bw=bw).fit(lr, weights=weights)
682
- # except ValueError:
683
- # return np.nan, None, None
684
-
685
- # # evaluate kde. grid needs to cover data.
686
- # end = p.dx * np.ceil((amax + p.dx) / p.dx)
687
- # grid = np.arange(0, end + p.dx / 2, p.dx)
688
- # grid_left = -grid[1:][::-1]
689
- # grid = np.concatenate((grid_left, grid))
690
- # nc = grid.shape[0] // 2
691
- # # nc = 0
692
- # assert grid.shape == (2 * nc + 1,)
693
- # assert grid[nc] == 0.0
694
- # ev = kde.evaluate(grid)
695
-
696
- # # find the HIGHEST local max right of 0, if any
697
- # peak_ix_right = _find_right_peak_index(ev, nc)
698
- # # find the NEAREST local max left or at 0, if any
699
- # peak_ix_left = _find_left_peak_index(ev, nc)
700
- # no_right_peak = peak_ix_right == ev.shape[0]
701
- # no_left_peak = peak_ix_left == -1
702
- # # make sure that we don't hit unreachable code in left peak
703
- # assert peak_ix_left != -2
704
- # # there has to be a peak. kdes can't ever be flat.
705
- # assert not (no_left_peak and no_right_peak)
706
-
707
- # if no_right_peak:
708
- # # no isolation
709
- # iso = 1.0
710
- # elif no_left_peak:
711
- # # good isolation
712
- # iso = 0.0
713
- # else:
714
- # # main case
715
- # assert peak_ix_right > peak_ix_left
716
- # rad = int(np.ceil(5 / p.dx))
717
- # dip = np.min(ev[max(nc - rad, peak_ix_left) : min(nc + rad + 1, peak_ix_right)])
718
- # peak = ev[peak_ix_right]
719
- # iso = dip / peak
720
-
721
- # if p.return_kdes:
722
- # # ev = ev[nc:]
723
- # if grid[-1] < p.kde_rhs:
724
- # domain = np.arange(-p.kde_rhs, p.kde_rhs + p.dx / 2, p.dx)
725
- # npad = domain.size - grid.size
726
- # assert npad % 2 == 0
727
- # kde = np.pad(ev, (npad // 2, npad // 2))
728
- # else:
729
- # mask = grid == grid.clip(-p.kde_rhs, p.kde_rhs)
730
- # kde = ev[mask]
731
- # domain = grid[mask]
732
- # else:
733
- # kde = domain = None
734
-
735
- # # compare value at 0 to peak value
736
- # return iso, domain, kde
737
-
738
-
739
636
  @numba.njit
740
637
  def _find_right_peak_index(x: np.ndarray, i: int) -> int:
741
638
  n = x.shape[0]
@@ -50,7 +50,7 @@ class DARTsortUserConfig:
50
50
  subsampling_spikes_per_channel: int | None = 5000
51
51
  """Detection steps before the final matching round will run until at least
52
52
  this many spikes are found or the whole recording is covered, to make sure
53
- that there is enough data for clustering. See also subsampling_fraction.
53
+ that there is enough data for clustering. See also subsampling_presence.
54
54
  Set to None to disable subsampling."""
55
55
 
56
56
  subsampling_presence: Annotated[float, Field(gt=0.0, le=1.0)] = 0.1
@@ -332,7 +332,7 @@ class DeveloperConfig(DARTsortUserConfig):
332
332
  n_neighbors_search: int | None = 50
333
333
 
334
334
  # filters
335
- gmm_isolation_threshold: float | None = 0.975
335
+ gmm_isolation_threshold: float | None = None
336
336
  collision_cleaning_error_threshold: float | None = 0.3
337
337
 
338
338
  # gaussian mixture high level
@@ -287,6 +287,7 @@ class DARTsortAnalysis:
287
287
  template_data=template_data,
288
288
  motion=motion,
289
289
  features_cfg=clustering_features_cfg,
290
+ compute_extensions_if_templates=None,
290
291
  )
291
292
  analyzer.compute("quality_metrics", metric_names=quality_metrics)
292
293
  qc_df = analyzer.get_extension("quality_metrics").get_data()