DARTsort 0.5.24__tar.gz → 0.5.25__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. {dartsort-0.5.24 → dartsort-0.5.25}/PKG-INFO +2 -2
  2. {dartsort-0.5.24 → dartsort-0.5.25}/README.md +1 -1
  3. {dartsort-0.5.24 → dartsort-0.5.25}/docs/index.md +1 -1
  4. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/clustering/mixture.py +161 -37
  5. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/config.py +24 -7
  6. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/detect/detect.py +113 -53
  7. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/evaluate/noise_recording_tools.py +7 -4
  8. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/evaluate/simlib.py +2 -1
  9. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/main.py +6 -11
  10. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/__init__.py +3 -4
  11. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/matching.py +1 -4
  12. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/matching_util/compressed_upsampled.py +1 -0
  13. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/peel_base.py +27 -5
  14. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/peel_lib.py +2 -450
  15. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/subtract.py +148 -37
  16. dartsort-0.5.25/src/dartsort/peel/subtract_util.py +1169 -0
  17. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/threshold.py +3 -3
  18. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/_multichan_denoiser_kit.py +17 -11
  19. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/all_transformers.py +5 -0
  20. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/amortized_localization.py +3 -1
  21. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/decollider.py +326 -44
  22. dartsort-0.5.25/src/dartsort/transform/matched_filter_net.py +282 -0
  23. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/pipeline.py +34 -12
  24. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/temporal_pca.py +50 -34
  25. dartsort-0.5.25/src/dartsort/transform/vq_matched_filter.py +157 -0
  26. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/data_util.py +12 -8
  27. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/internal_config.py +438 -298
  28. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/main_util.py +12 -36
  29. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/nn_util.py +65 -10
  30. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/preprocess_util.py +29 -10
  31. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/py_util.py +2 -2
  32. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/spiketorch.py +58 -0
  33. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/waveform_util.py +27 -0
  34. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/analysis_plots.py +63 -20
  35. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/vismain.py +2 -1
  36. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort.egg-info/PKG-INFO +2 -2
  37. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort.egg-info/SOURCES.txt +3 -0
  38. {dartsort-0.5.24 → dartsort-0.5.25}/tests/conftest.py +61 -0
  39. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_alignment.py +3 -1
  40. dartsort-0.5.25/tests/test_config.py +102 -0
  41. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_dartsort.py +1 -1
  42. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_matching.py +4 -3
  43. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_mixture_classifier.py +2 -0
  44. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_modeling.py +19 -19
  45. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_spiketorch.py +44 -0
  46. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_subtract.py +107 -3
  47. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_templates.py +2 -2
  48. dartsort-0.5.24/tests/test_config.py +0 -27
  49. {dartsort-0.5.24 → dartsort-0.5.25}/.github/workflows/ci.yml +0 -0
  50. {dartsort-0.5.24 → dartsort-0.5.25}/.github/workflows/ci_windows.yml +0 -0
  51. {dartsort-0.5.24 → dartsort-0.5.25}/.github/workflows/deploy.yml +0 -0
  52. {dartsort-0.5.24 → dartsort-0.5.25}/.github/workflows/docs.yml +0 -0
  53. {dartsort-0.5.24 → dartsort-0.5.25}/.gitignore +0 -0
  54. {dartsort-0.5.24 → dartsort-0.5.25}/docs/main_api.md +0 -0
  55. {dartsort-0.5.24 → dartsort-0.5.25}/docs/toolkit_api.md +0 -0
  56. {dartsort-0.5.24 → dartsort-0.5.25}/environment.yml +0 -0
  57. {dartsort-0.5.24 → dartsort-0.5.25}/notebook/.keep +0 -0
  58. {dartsort-0.5.24 → dartsort-0.5.25}/notebook/fit_and_batch_matching.py +0 -0
  59. {dartsort-0.5.24 → dartsort-0.5.25}/notebook/ibl_dandi_example.py +0 -0
  60. {dartsort-0.5.24 → dartsort-0.5.25}/pyproject.toml +0 -0
  61. {dartsort-0.5.24 → dartsort-0.5.25}/setup.cfg +0 -0
  62. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/__init__.py +0 -0
  63. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/__main__.py +0 -0
  64. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/cli.py +0 -0
  65. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/clustering/__init__.py +0 -0
  66. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/clustering/agglomerate.py +0 -0
  67. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/clustering/cluster_util.py +0 -0
  68. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/clustering/clustering.py +0 -0
  69. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/clustering/clustering_features.py +0 -0
  70. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/clustering/density.py +0 -0
  71. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/clustering/forward_backward.py +0 -0
  72. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/clustering/kmeans.py +0 -0
  73. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/clustering/merge.py +0 -0
  74. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/clustering/refine_util.py +0 -0
  75. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/detect/__init__.py +0 -0
  76. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/evaluate/__init__.py +0 -0
  77. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/evaluate/analysis.py +0 -0
  78. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/evaluate/comparison.py +0 -0
  79. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/evaluate/hybrid_util.py +0 -0
  80. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/evaluate/hybridkit.py +0 -0
  81. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  82. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/evaluate/simkit.py +0 -0
  83. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/localize/__init__.py +0 -0
  84. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/localize/localize_torch.py +0 -0
  85. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/localize/localize_util.py +0 -0
  86. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/grab.py +0 -0
  87. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/matching_util/__init__.py +0 -0
  88. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/matching_util/drifty.py +0 -0
  89. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  90. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  91. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  92. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/reduction_template.py +0 -0
  93. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/peel/shaver.py +0 -0
  94. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  95. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/pretrained/detect_np1.pt +0 -0
  96. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/pretrained/detect_np2.pt +0 -0
  97. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  98. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  99. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  100. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  101. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  102. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  103. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/templates/__init__.py +0 -0
  104. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/templates/get_templates.py +0 -0
  105. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/templates/postprocess_util.py +0 -0
  106. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/templates/realignment.py +0 -0
  107. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/templates/template_util.py +0 -0
  108. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/templates/templates.py +0 -0
  109. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/templates/templib.py +0 -0
  110. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/__init__.py +0 -0
  111. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/amplitudes.py +0 -0
  112. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/denoising_scorer.py +0 -0
  113. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/enforce_decrease.py +0 -0
  114. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/fixed_prop.py +0 -0
  115. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/interp.py +0 -0
  116. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/localize.py +0 -0
  117. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/matching_denoiser.py +0 -0
  118. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/mixture_classifier.py +0 -0
  119. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/reduction.py +0 -0
  120. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  121. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/supervised_denoiser.py +0 -0
  122. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/transform_base.py +0 -0
  123. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/transform/whiten.py +0 -0
  124. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/__init__.py +0 -0
  125. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/cli_util.py +0 -0
  126. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/drift_util.py +0 -0
  127. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/interpolation_util.py +0 -0
  128. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/job_util.py +0 -0
  129. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/logging_util.py +0 -0
  130. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/more_operators.py +0 -0
  131. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/motion.py +0 -0
  132. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/multiprocessing_util.py +0 -0
  133. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/noise_util.py +0 -0
  134. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/peel_util.py +0 -0
  135. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/registration_util.py +0 -0
  136. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/residual_util.py +0 -0
  137. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/sparse_util.py +0 -0
  138. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/spikeio.py +0 -0
  139. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/testing_util/__init__.py +0 -0
  140. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  141. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  142. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/torch_optimization_util.py +0 -0
  143. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/util/torch_util.py +0 -0
  144. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/__init__.py +0 -0
  145. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/colors.py +0 -0
  146. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/gt.py +0 -0
  147. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/layout.py +0 -0
  148. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/metrics_util.py +0 -0
  149. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/mixture.py +0 -0
  150. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/recanim.py +0 -0
  151. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/scatterplots.py +0 -0
  152. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/sorting.py +0 -0
  153. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/unit.py +0 -0
  154. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/unit_comparison.py +0 -0
  155. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/versus.py +0 -0
  156. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort/vis/waveforms.py +0 -0
  157. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort.egg-info/dependency_links.txt +0 -0
  158. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort.egg-info/entry_points.txt +0 -0
  159. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort.egg-info/requires.txt +0 -0
  160. {dartsort-0.5.24 → dartsort-0.5.25}/src/dartsort.egg-info/top_level.txt +0 -0
  161. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_agglomerate.py +0 -0
  162. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_cluster_util.py +0 -0
  163. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_clustering.py +0 -0
  164. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_clustering_features.py +0 -0
  165. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_data_util.py +0 -0
  166. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_density.py +0 -0
  167. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_detect.py +0 -0
  168. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_drift_util.py +0 -0
  169. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_drifty_matching.py +0 -0
  170. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_enforce_decrease.py +0 -0
  171. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_grab_and_featurize.py +0 -0
  172. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_hybridkit.py +0 -0
  173. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_kmeans.py +0 -0
  174. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_linalg.py +0 -0
  175. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_localize_torch.py +0 -0
  176. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_simkit.py +0 -0
  177. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_threshold.py +0 -0
  178. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_torch_optimization_util.py +0 -0
  179. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_transform.py +0 -0
  180. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_util.py +0 -0
  181. {dartsort-0.5.24 → dartsort-0.5.25}/tests/test_waveform_util.py +0 -0
  182. {dartsort-0.5.24 → dartsort-0.5.25}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.24
3
+ Version: 0.5.25
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -48,7 +48,7 @@ Requires-Dist: matplotlib-venn; extra == "vis"
48
48
  [![coveralls](https://coveralls.io/repos/github/cwindolf/dartsort/badge.svg?branch=main)](https://coveralls.io/github/cwindolf/dartsort)
49
49
  [![Zenodo DOI](https://zenodo.org/badge/421108722.svg)](https://doi.org/10.5281/zenodo.16943074)
50
50
  [![pypi: dartsort](https://img.shields.io/pypi/v/dartsort?label=pypi:%20dartsort)](https://pypi.org/p/dartsort)
51
- [![zulip: dartsort](https://img.shields.io/badge/zulip-dartsort-brightgreen.svg)](https://dartsort.zulip.org)
51
+ [![zulip: dartsort](https://img.shields.io/badge/zulip-dartsort-brightgreen.svg)](https://dartsort.zulipchat.org)
52
52
 
53
53
  # dartsort
54
54
 
@@ -5,7 +5,7 @@
5
5
  [![coveralls](https://coveralls.io/repos/github/cwindolf/dartsort/badge.svg?branch=main)](https://coveralls.io/github/cwindolf/dartsort)
6
6
  [![Zenodo DOI](https://zenodo.org/badge/421108722.svg)](https://doi.org/10.5281/zenodo.16943074)
7
7
  [![pypi: dartsort](https://img.shields.io/pypi/v/dartsort?label=pypi:%20dartsort)](https://pypi.org/p/dartsort)
8
- [![zulip: dartsort](https://img.shields.io/badge/zulip-dartsort-brightgreen.svg)](https://dartsort.zulip.org)
8
+ [![zulip: dartsort](https://img.shields.io/badge/zulip-dartsort-brightgreen.svg)](https://dartsort.zulipchat.org)
9
9
 
10
10
  # dartsort
11
11
 
@@ -5,7 +5,7 @@
5
5
  [![coveralls](https://coveralls.io/repos/github/cwindolf/dartsort/badge.svg?branch=main)](https://coveralls.io/github/cwindolf/dartsort)
6
6
  [![Zenodo DOI](https://zenodo.org/badge/421108722.svg)](https://doi.org/10.5281/zenodo.16943074)
7
7
  [![pypi: dartsort](https://img.shields.io/pypi/v/dartsort?label=pypi:%20dartsort)](https://pypi.org/p/dartsort)
8
- [![zulip: dartsort](https://img.shields.io/badge/zulip-dartsort-brightgreen.svg)](https://dartsort.zulip.org)
8
+ [![zulip: dartsort](https://img.shields.io/badge/zulip-dartsort-brightgreen.svg)](https://dartsort.zulipchat.org)
9
9
 
10
10
  # dartsort
11
11
 
@@ -91,16 +91,19 @@ from ..util.py_util import databag, panic
91
91
  from ..util.spiketorch import (
92
92
  _nonzero_static,
93
93
  cosine_distance,
94
+ csr_members_ordered,
94
95
  ecl,
95
96
  entropy,
96
97
  mean_elbo_dim1,
97
98
  normeuc_distance,
98
99
  scaled_normeuc_distance,
99
100
  sign,
101
+ sort_to_csr,
100
102
  spawn_torch_rg,
103
+ stratified_subsample,
101
104
  )
102
105
  from ..util.torch_util import BModule, cleanup_and_log_gpu_usage, torch_compiler
103
- from .cluster_util import decrumb, linkage, maximal_leaf_groups
106
+ from .cluster_util import decrumb, linkage, maximal_leaf_groups, violation_statistics
104
107
  from .clustering_features import StableWaveformFeatures
105
108
  from .kmeans import _one_gumbel_nolog, batched_kmeans, kmeans
106
109
 
@@ -198,7 +201,14 @@ def tmm_demix(
198
201
  min_iters=tmm.p.main_min_iters,
199
202
  )
200
203
  elif step_type == "merge":
201
- run_merge(tmm, train_data, val_data, prog_level)
204
+ run_merge(
205
+ tmm,
206
+ train_data,
207
+ val_data,
208
+ prog_level,
209
+ full_data=full_data,
210
+ original_sorting=sorting,
211
+ )
202
212
  tmm.em(
203
213
  train_data,
204
214
  show_progress=prog_level,
@@ -830,6 +840,11 @@ class TMMParams:
830
840
  fix_responsibilities: bool
831
841
  scale_dist_args: tuple[float, float, float]
832
842
  whiten_dist: bool
843
+ merge_violation_threshold: float | None
844
+ merge_violation_ms: float
845
+ merge_jitter_ms: float
846
+ merge_censor_ms: float
847
+ merge_min_violation_evidence: float
833
848
 
834
849
  @classmethod
835
850
  def from_refinement_cfg(cls, refinement_cfg: RefinementConfig):
@@ -867,6 +882,11 @@ class TMMParams:
867
882
  fix_responsibilities=refinement_cfg.fix_responsibilities_in_split,
868
883
  scale_dist_args=refinement_cfg.scale_dist_args,
869
884
  whiten_dist=refinement_cfg.whiten_dist,
885
+ merge_violation_threshold=refinement_cfg.glom_violation_threshold,
886
+ merge_violation_ms=refinement_cfg.glom_violation_ms,
887
+ merge_jitter_ms=refinement_cfg.glom_jitter_ms,
888
+ merge_censor_ms=refinement_cfg.censor_ms,
889
+ merge_min_violation_evidence=refinement_cfg.glom_min_violation_evidence,
870
890
  )
871
891
 
872
892
 
@@ -1895,6 +1915,8 @@ class TruncatedSpikeData(BatchedSpikeData):
1895
1915
  gen: torch.Generator,
1896
1916
  labels: Tensor | None,
1897
1917
  min_count: int = 0,
1918
+ min_per_neighborhood: int = 0,
1919
+ min_per_unit: int = 0,
1898
1920
  ) -> DenseSpikeData | None:
1899
1921
  assert self.candidates is not None
1900
1922
  assert unit_ids is not None
@@ -1915,9 +1937,30 @@ class TruncatedSpikeData(BatchedSpikeData):
1915
1937
  (ixs,) = mask.nonzero(as_tuple=True)
1916
1938
  if min_count and ixs.numel() < min_count:
1917
1939
  return None
1918
- if (nixs := ixs.numel()) > self.dense_slice_size_per_unit * n_ids:
1940
+ n_target = self.dense_slice_size_per_unit * n_ids
1941
+ nixs = ixs.numel()
1942
+ assert not (min_per_neighborhood and min_per_unit)
1943
+ if nixs > n_target and min_per_neighborhood:
1944
+ ixs = stratified_subsample(
1945
+ ixs,
1946
+ self.neighborhood_ids[ixs],
1947
+ n_groups=self.neighborhoods.n_neighborhoods,
1948
+ n_target=n_target,
1949
+ min_per_group=min_per_neighborhood,
1950
+ gen=gen,
1951
+ )
1952
+ elif nixs > n_target and min_per_unit:
1953
+ ixs = stratified_subsample(
1954
+ ixs,
1955
+ labels[ixs],
1956
+ n_groups=int(unit_ids.max()) + 1,
1957
+ n_target=n_target,
1958
+ min_per_group=min_per_unit,
1959
+ gen=gen,
1960
+ )
1961
+ elif nixs > n_target:
1919
1962
  perm = torch.randperm(nixs, generator=gen, device=ixs.device)
1920
- ixs = ixs[perm[: self.dense_slice_size_per_unit * n_ids]]
1963
+ ixs = ixs[perm[:n_target]]
1921
1964
  ixs = torch.msort(ixs)
1922
1965
 
1923
1966
  return self.dense_slice(ixs)
@@ -2001,17 +2044,28 @@ class TruncatedSpikeData(BatchedSpikeData):
2001
2044
  split_mask: Tensor,
2002
2045
  split_labels: Tensor,
2003
2046
  distances: Tensor,
2004
- ) -> NeighborhoodLUT:
2005
- """Erase candidates for spikes in the split, replace labels, re-bootstrap."""
2047
+ ) -> tuple[NeighborhoodLUT, int]:
2048
+ """Erase candidates for spikes in the split, replace labels, re-bootstrap.
2049
+
2050
+ Neighborhoods left uncovered by the split get new units, and their count is returned.
2051
+ """
2006
2052
  assert self.candidates is not None
2007
2053
  self.candidates[:, 0].masked_fill_(unit_mask, -1)
2008
2054
  (split_ix,) = split_mask.nonzero(as_tuple=True)
2009
2055
  self.candidates[split_ix, 0] = split_labels[split_ix]
2010
2056
 
2057
+ n_units = distances.shape[0]
2058
+ n_new = self.ensure_coverage(allow_new_units=True, next_label=n_units)
2059
+ if n_new:
2060
+ d = distances.new_full((n_units + n_new, n_units + n_new), torch.inf)
2061
+ d[:n_units, :n_units] = distances
2062
+ d[n_units:, n_units:].fill_diagonal_(0.0)
2063
+ distances = d
2064
+
2011
2065
  # have to do a full bootstrap, bc it's hard to figure out what to do with
2012
2066
  # spikes whose candidates contain the units that were split. this way, the
2013
2067
  # lut invariants are maintained, and at least the top labels are the same.
2014
- return self.bootstrap_candidates(distances, allow_uncovered=True)
2068
+ return self.bootstrap_candidates(distances), n_new
2015
2069
 
2016
2070
  def full_proposal_view(self, un_adj_lut: NeighborhoodLUT):
2017
2071
  return FullProposalDataView.from_truncated_spike_data(self, un_adj_lut)
@@ -3085,7 +3139,11 @@ class TruncatedMixtureModel(BaseMixtureModel):
3085
3139
 
3086
3140
  # get dense train set slice in group
3087
3141
  split_data = train_data.dense_slice_by_unit(
3088
- group, gen=self.rg, min_count=self.p.min_count, labels=train_labels
3142
+ group,
3143
+ gen=self.rg,
3144
+ min_count=self.p.min_count,
3145
+ labels=train_labels,
3146
+ min_per_neighborhood=self.p.min_channel_count,
3089
3147
  )
3090
3148
  if split_data is None and debug:
3091
3149
  return None, SplitCaseDebugInfo(bailed=True, bail_reason="count")
@@ -3434,7 +3492,7 @@ class TruncatedMixtureModel(BaseMixtureModel):
3434
3492
 
3435
3493
  view = self.unit_slice(group)
3436
3494
  group_train_data = train_data.dense_slice_by_unit(
3437
- group, gen=self.rg, labels=train_labels
3495
+ group, gen=self.rg, labels=train_labels, min_per_unit=self.p.min_count
3438
3496
  )
3439
3497
  assert group_train_data is not None
3440
3498
  if pnoid:
@@ -3470,6 +3528,7 @@ class TruncatedMixtureModel(BaseMixtureModel):
3470
3528
  train_scores: Scores,
3471
3529
  eval_scores: Scores,
3472
3530
  *,
3531
+ pair_mask: Tensor | None = None,
3473
3532
  show_progress: bool = True,
3474
3533
  ) -> UnitRemapping:
3475
3534
  """Break into subgroups and merge those. This will also modify data objects."""
@@ -3490,6 +3549,10 @@ class TruncatedMixtureModel(BaseMixtureModel):
3490
3549
  groups = progbar(groups, desc="Merge", smoothing=0.0)
3491
3550
 
3492
3551
  for group in groups:
3552
+ group_pair_mask = None
3553
+ if pair_mask is not None:
3554
+ group_pair_mask = pair_mask[group[:, None], group[None, :]]
3555
+
3493
3556
  group_res = self.try_merge_group(
3494
3557
  group=group,
3495
3558
  train_data=train_data,
@@ -3497,6 +3560,7 @@ class TruncatedMixtureModel(BaseMixtureModel):
3497
3560
  eval_scores=eval_scores,
3498
3561
  train_labels=train_labels,
3499
3562
  eval_labels=eval_labels,
3563
+ pair_mask=group_pair_mask,
3500
3564
  )
3501
3565
 
3502
3566
  # no-merge cases
@@ -3523,6 +3587,13 @@ class TruncatedMixtureModel(BaseMixtureModel):
3523
3587
  # log(new_props) = props[group].sum() * sub_props)
3524
3588
  group_log_prop = self.b.log_proportions[group].logsumexp(dim=0)
3525
3589
  new_log_props = group_res.sub_proportions.log().add_(group_log_prop)
3590
+ new_log_props = new_log_props.clamp_(min=self.LP_MIN)
3591
+ if pnoid and not new_log_props[groups_kept].isfinite().all():
3592
+ raise AssertionError(
3593
+ f"Non-finite log proportions in merge of group {group.tolist()} "
3594
+ f"with partition {group_ids.tolist()}: {new_log_props=}, "
3595
+ f"{group_res.sub_proportions=}, {group_log_prop=}."
3596
+ )
3526
3597
 
3527
3598
  # demolish case. throw these away.
3528
3599
  ids_discard = group[group_ids == -1]
@@ -3538,6 +3609,8 @@ class TruncatedMixtureModel(BaseMixtureModel):
3538
3609
  # else, move component to first id in group and throw rest away
3539
3610
  first = ids_in_group[0]
3540
3611
  rest = ids_in_group[1:]
3612
+ if pnoid:
3613
+ assert first == ids_in_group.min()
3541
3614
 
3542
3615
  # first retains id (for now, cleanup below remaps to flat)
3543
3616
  result_map.mapping[ids_in_group] = first
@@ -3889,6 +3962,7 @@ class TruncatedMixtureModel(BaseMixtureModel):
3889
3962
  self.b.bases.resize_(new_n_units, *self.b.bases.shape[1:])
3890
3963
  self.b.bases[self.n_units :].zero_()
3891
3964
  self.n_units = new_n_units
3965
+ self.unit_ids = torch.arange(new_n_units)
3892
3966
 
3893
3967
  def _apply_splits(
3894
3968
  self,
@@ -4052,9 +4126,10 @@ class TruncatedMixtureModel(BaseMixtureModel):
4052
4126
  if pnoid:
4053
4127
  assert discard.mapping.max() < distances.shape[0]
4054
4128
  assert train_labels.max() < distances.shape[0]
4055
- lut = train_data.update_from_split(
4129
+ lut, n_new = train_data.update_from_split(
4056
4130
  train_candidate_mask, train_labels_mask, train_labels, distances
4057
4131
  )
4132
+ self.add_new_blank_units(n_new)
4058
4133
  self.update_lut(lut)
4059
4134
  assert self.lut_params is not None
4060
4135
  self.lut_params.check()
@@ -4641,9 +4716,9 @@ def instantiate_and_bootstrap_tmm(
4641
4716
  save_kwargs: dict | None = None,
4642
4717
  ) -> MixtureModelAndDatasets:
4643
4718
  global pnoid
4644
- pnoid = logger.isEnabledFor(DARTSORTVERBOSE)
4719
+ pnoid = pnoid or logger.isEnabledFor(DARTSORTVERBOSE)
4645
4720
  if pnoid:
4646
- logger.dartsortverbose("Extra TMM asserts are on.")
4721
+ logger.info("Extra TMM asserts are on.")
4647
4722
 
4648
4723
  rg = np.random.default_rng(seed)
4649
4724
  computation_cfg = ensure_computation_config(computation_cfg)
@@ -4831,7 +4906,7 @@ def truncated_kmeanspp(
4831
4906
  cent_visible = visible[:, cent_neighbs]
4832
4907
 
4833
4908
  full_labels = torch.full(x.shape[:1], -1, dtype=torch.long, device=device)
4834
- order, indptr, _ = _sort_to_compressed_neighborhood_sparse(
4909
+ order, indptr, _ = sort_to_csr(
4835
4910
  neighb_ids, neighborhoods.n_neighborhoods
4836
4911
  )
4837
4912
  indptr_list = indptr.tolist()
@@ -4921,7 +4996,7 @@ def _truncated_kmeanspp_inner(
4921
4996
  # Initialize
4922
4997
  train_centroid_ixs = X.new_full((max_k,), n, dtype=torch.long)
4923
4998
  distsq = X.new_full((n,), torch.inf)
4924
- order, indptr, counts = _sort_to_compressed_neighborhood_sparse(
4999
+ order, indptr, counts = sort_to_csr(
4925
5000
  Xneighbixs, n_neighb
4926
5001
  )
4927
5002
 
@@ -5046,7 +5121,7 @@ def _truncated_kmeanspp_propose(
5046
5121
  ) -> tuple[Tensor, Tensor | None]:
5047
5122
  cent_neighb = Xneighbixs[centroid_ix]
5048
5123
  (vis_neighbs,) = visible[cent_neighb].nonzero(as_tuple=True)
5049
- ix = _neighborhood_members_ordered(order, indptr, counts, vis_neighbs)
5124
+ ix = csr_members_ordered(order, indptr, counts, vis_neighbs)
5050
5125
  if not ix.numel():
5051
5126
  return ix, None
5052
5127
 
@@ -5070,27 +5145,6 @@ def _truncated_kmeanspp_commit_(distsq: Tensor, ix: Tensor, d: Tensor) -> None:
5070
5145
  distsq[ix] = d
5071
5146
 
5072
5147
 
5073
- def _sort_to_compressed_neighborhood_sparse(
5074
- neighborhood_ids: Tensor, n_neighborhoods: int
5075
- ) -> tuple[Tensor, Tensor, Tensor]:
5076
- counts = torch.bincount(neighborhood_ids, minlength=n_neighborhoods)
5077
- order = torch.argsort(neighborhood_ids, stable=True)
5078
- indptr = F.pad(counts.cumsum(0), (1, 0))
5079
- return order, indptr, counts
5080
-
5081
-
5082
- def _neighborhood_members_ordered(
5083
- order: Tensor, indptr: Tensor, counts: Tensor, neighbs: Tensor
5084
- ) -> Tensor:
5085
- sizes = counts[neighbs]
5086
- total = int(sizes.sum())
5087
- if not total:
5088
- return order[:0]
5089
- # ragged arange over the selected CSR rows
5090
- starts = torch.repeat_interleave(indptr[neighbs] - sizes.cumsum(0) + sizes, sizes)
5091
- return order[starts + torch.arange(total, device=order.device)]
5092
-
5093
-
5094
5148
  def run_split(
5095
5149
  tmm: TruncatedMixtureModel,
5096
5150
  *,
@@ -5140,7 +5194,15 @@ def run_merge(
5140
5194
  train_data: TruncatedSpikeData,
5141
5195
  val_data: TruncatedSpikeData | None,
5142
5196
  prog_level: int,
5197
+ full_data: BatchedSpikeData | None = None,
5198
+ original_sorting: DARTsortSorting | None = None,
5143
5199
  ):
5200
+ pair_mask = violation_pair_mask(
5201
+ tmm=tmm,
5202
+ full_data=full_data,
5203
+ original_sorting=original_sorting,
5204
+ prog_level=prog_level,
5205
+ )
5144
5206
  train_scores = tmm.soft_assign(
5145
5207
  data=train_data,
5146
5208
  full_proposal_view=True,
@@ -5161,11 +5223,69 @@ def run_merge(
5161
5223
  val_data,
5162
5224
  eval_scores=eval_scores,
5163
5225
  train_scores=train_scores,
5226
+ pair_mask=pair_mask,
5164
5227
  show_progress=prog_level > 0,
5165
5228
  )
5166
5229
  logger.info(f"Merge {merge_map.mapping.shape[0]} -> {merge_map.nuniq()} units.")
5167
5230
 
5168
5231
 
5232
+ def violation_pair_mask(
5233
+ *,
5234
+ tmm: TruncatedMixtureModel,
5235
+ full_data: BatchedSpikeData | None,
5236
+ original_sorting: DARTsortSorting | None,
5237
+ prog_level: int,
5238
+ ) -> Tensor | None:
5239
+ """Which pairs of units are allowed to merge by the refractory violation check
5240
+
5241
+ Units are assigned over the whole dataset, rather than just the training
5242
+ subset, so that the violation counts get as much evidence as possible.
5243
+ Pairs whose jittered chance level is too small to say anything are filled
5244
+ with a ratio of 0, which allows their merge.
5245
+ """
5246
+ if tmm.p.merge_violation_threshold is None:
5247
+ return None
5248
+ if full_data is None or original_sorting is None:
5249
+ return None
5250
+
5251
+ full_scores = tmm.soft_assign(
5252
+ data=full_data,
5253
+ full_proposal_view=True,
5254
+ needs_bootstrap=False,
5255
+ show_progress=prog_level - 1,
5256
+ )
5257
+ sorting = original_sorting.ephemeral_replace(labels=labels_from_scores(full_scores))
5258
+ violation = violation_statistics(
5259
+ sorting,
5260
+ censor_ms=tmm.p.merge_censor_ms,
5261
+ viol_ms=tmm.p.merge_violation_ms,
5262
+ jitter_ms=tmm.p.merge_jitter_ms,
5263
+ )
5264
+ assert violation.jitter_counts is not None
5265
+ ratio = violation.jitter_viol_ratio(
5266
+ tmm.p.merge_min_violation_evidence, fill_value=0.0
5267
+ )
5268
+
5269
+ # violation_statistics sizes its matrices by the largest label present, so
5270
+ # dead units at the end of the id range are missing. they can merge.
5271
+ allowed = np.ones((tmm.n_units, tmm.n_units), dtype=bool)
5272
+ nu = ratio.shape[0]
5273
+ assert nu <= tmm.n_units
5274
+ allowed[:nu, :nu] = ratio < tmm.p.merge_violation_threshold
5275
+
5276
+ # singleton parts are checked against the diagonal in allowed_partitions
5277
+ np.fill_diagonal(allowed, True)
5278
+
5279
+ n_blocked = np.triu(np.logical_not(allowed), k=1).sum()
5280
+ logger.dartsortdebug(
5281
+ "Violation check blocks %d of %d unit pairs from merging.",
5282
+ n_blocked,
5283
+ (tmm.n_units * (tmm.n_units - 1)) // 2,
5284
+ )
5285
+
5286
+ return torch.from_numpy(allowed)
5287
+
5288
+
5169
5289
  def save_tmm_labels(
5170
5290
  *,
5171
5291
  tmm: TruncatedMixtureModel,
@@ -5663,6 +5783,10 @@ def all_demolished_partitions(
5663
5783
  assert (
5664
5784
  can_demolish_mask.shape == part.unit_ids.shape == part.group_ids.shape
5665
5785
  )
5786
+ if part.subset_ids:
5787
+ allp.append(part)
5788
+ continue
5789
+
5666
5790
  single_ixs = torch.tensor(part.single_ixs, dtype=torch.long)
5667
5791
  npart = single_ixs.numel()
5668
5792
  (part_demo_ix,) = can_demolish_mask[single_ixs].nonzero(as_tuple=True)
@@ -6875,7 +6999,7 @@ def _fill_blank_labels(
6875
6999
  if pnoid:
6876
7000
  Kold = labels.max().item() + 1 + int(Nblank > 0)
6877
7001
  _lshp = labels.unique().shape
6878
- assert _lshp == (Kold,), f"{Kold=} {_lshp=}"
7002
+ assert _lshp[0] <= Kold, f"{Kold=} {_lshp=}"
6879
7003
  for j, nid in enumerate(uncovered_neighbs):
6880
7004
  (in_nid,) = (neighborhood_ids == nid).nonzero(as_tuple=True)
6881
7005
  n_spikes += in_nid.numel()
@@ -32,13 +32,26 @@ class DARTsortUserConfig:
32
32
  do_motion_estimation: bool = True
33
33
  """Set this to false if your data is super stable or already motion-corrected."""
34
34
 
35
- preprocessing: PreprocessingStrategy = "none"
36
- """If other than `'none'`, dartsort will apply some preprocessing to the
37
- recording. Leave as `'none'` if you are passing in an already-preprocesed
38
- recording. If so, be aware that dartsort expects its input to be standardized on
39
- each channel in addition to the usual highpass filtering, but that
40
- whitening is handled internally. See util/preprocess_util.py if you're
41
- curious about the details of the methods.
35
+ already_preprocessed: Literal["yes", "no", "assume_yes_if_float"] = (
36
+ "assume_yes_if_float"
37
+ )
38
+ """Is your data already preprocessed?
39
+
40
+ If so, the preprocessing flag below is ignored. By default, the recording is
41
+ assumed to be preprocessed already (including standardization) if its data
42
+ type is any floating point type.
43
+
44
+ dartsort will warn you if the data looks weird according to some simple checks
45
+ (see check_recording in data_util and warn_about_preprocessing in
46
+ preprocess_util).
47
+ """
48
+
49
+ preprocessing: PreprocessingStrategy = "ibllikecmr"
50
+ """If other than `'none'`, and following the already_preprocessed flag above,
51
+ dartsort will apply some preprocessing to the recording. Be aware that
52
+ dartsort expects its input to be standardized on each channel in addition to
53
+ the usual highpass filtering, but that whitening is handled internally. See
54
+ util/preprocess_util.py if you're curious about the details of the methods.
42
55
 
43
56
  Options: `'ibllikecmr', 'ibllike', 'standardize', 'none'`
44
57
  """
@@ -263,6 +276,10 @@ class DeveloperConfig(DARTsortUserConfig):
263
276
  nn_denoiser_max_waveforms_fit: int = 512_000
264
277
  nn_denoiser_noise_waveforms: int = 100 * 256
265
278
  nn_denoiser_extra_kwargs: dict | None = None
279
+ score_filter_radius_um: float = 0.0
280
+ detection_proposal: Literal["voltage", "tpca", "vq", "score_net"] = "voltage"
281
+ proposal_filters: int = 1
282
+ score_proposal_threshold: float = 5.0
266
283
  do_tpca_denoise: bool = True
267
284
  first_denoiser_thinning: float = 0.0
268
285
  first_denoiser_spatial_dedup_radius: float = 100.0