DARTsort 0.5.21__tar.gz → 0.5.23__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. {dartsort-0.5.21 → dartsort-0.5.23}/.gitignore +1 -0
  2. {dartsort-0.5.21 → dartsort-0.5.23}/PKG-INFO +1 -1
  3. {dartsort-0.5.21 → dartsort-0.5.23}/pyproject.toml +1 -2
  4. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/__init__.py +6 -1
  5. dartsort-0.5.23/src/dartsort/__main__.py +4 -0
  6. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/cli.py +8 -4
  7. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/clustering/agglomerate.py +34 -20
  8. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/clustering/cluster_util.py +158 -46
  9. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/clustering/clustering.py +3 -2
  10. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/clustering/clustering_features.py +82 -103
  11. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/clustering/density.py +6 -6
  12. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/clustering/merge.py +32 -6
  13. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/clustering/mixture.py +25 -22
  14. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/clustering/refine_util.py +1 -4
  15. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/config.py +2 -1
  16. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/evaluate/analysis.py +27 -29
  17. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/evaluate/comparison.py +34 -23
  18. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/evaluate/hybrid_util.py +22 -16
  19. dartsort-0.5.23/src/dartsort/evaluate/hybridkit.py +286 -0
  20. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/evaluate/noise_recording_tools.py +1 -1
  21. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/evaluate/sim_template_tools.py +295 -102
  22. dartsort-0.5.23/src/dartsort/evaluate/simkit.py +372 -0
  23. dartsort-0.5.23/src/dartsort/evaluate/simlib.py +1063 -0
  24. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/main.py +103 -80
  25. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/matching.py +3 -8
  26. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/peel_base.py +5 -12
  27. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/peel_lib.py +3 -4
  28. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/reduction_template.py +5 -9
  29. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/subtract.py +1 -1
  30. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/templates/postprocess_util.py +8 -8
  31. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/templates/template_util.py +2 -2
  32. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/templates/templates.py +14 -16
  33. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/amplitudes.py +3 -0
  34. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/mixture_classifier.py +4 -3
  35. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/pipeline.py +6 -5
  36. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/reduction.py +4 -6
  37. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/transform_base.py +5 -2
  38. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/cli_util.py +2 -2
  39. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/data_util.py +365 -107
  40. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/internal_config.py +35 -22
  41. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/interpolation_util.py +18 -14
  42. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/logging_util.py +8 -8
  43. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/main_util.py +58 -18
  44. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/motion.py +52 -7
  45. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/multiprocessing_util.py +2 -2
  46. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/noise_util.py +2 -2
  47. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/peel_util.py +10 -16
  48. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/py_util.py +12 -7
  49. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/torch_util.py +1 -4
  50. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/gt.py +90 -78
  51. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/scatterplots.py +7 -5
  52. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/sorting.py +2 -1
  53. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/unit_comparison.py +21 -18
  54. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort.egg-info/PKG-INFO +1 -1
  55. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort.egg-info/SOURCES.txt +3 -0
  56. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_alignment.py +6 -3
  57. dartsort-0.5.23/tests/test_cluster_util.py +209 -0
  58. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_clustering.py +3 -2
  59. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_data_util.py +1 -1
  60. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_grab_and_featurize.py +3 -4
  61. dartsort-0.5.23/tests/test_hybridkit.py +432 -0
  62. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_matching.py +1 -1
  63. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_simkit.py +98 -22
  64. dartsort-0.5.21/src/dartsort/evaluate/simkit.py +0 -911
  65. dartsort-0.5.21/src/dartsort/evaluate/simlib.py +0 -362
  66. dartsort-0.5.21/tests/test_cluster_util.py +0 -83
  67. {dartsort-0.5.21 → dartsort-0.5.23}/.github/workflows/ci.yml +0 -0
  68. {dartsort-0.5.21 → dartsort-0.5.23}/.github/workflows/ci_windows.yml +0 -0
  69. {dartsort-0.5.21 → dartsort-0.5.23}/.github/workflows/deploy.yml +0 -0
  70. {dartsort-0.5.21 → dartsort-0.5.23}/.github/workflows/docs.yml +0 -0
  71. {dartsort-0.5.21 → dartsort-0.5.23}/README.md +0 -0
  72. {dartsort-0.5.21 → dartsort-0.5.23}/docs/index.md +0 -0
  73. {dartsort-0.5.21 → dartsort-0.5.23}/docs/main_api.md +0 -0
  74. {dartsort-0.5.21 → dartsort-0.5.23}/docs/toolkit_api.md +0 -0
  75. {dartsort-0.5.21 → dartsort-0.5.23}/environment.yml +0 -0
  76. {dartsort-0.5.21 → dartsort-0.5.23}/notebook/.keep +0 -0
  77. {dartsort-0.5.21 → dartsort-0.5.23}/notebook/ibl_dandi_example.py +0 -0
  78. {dartsort-0.5.21 → dartsort-0.5.23}/setup.cfg +0 -0
  79. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/clustering/__init__.py +0 -0
  80. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/clustering/forward_backward.py +0 -0
  81. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/clustering/kmeans.py +0 -0
  82. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/detect/__init__.py +0 -0
  83. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/detect/detect.py +0 -0
  84. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/evaluate/__init__.py +0 -0
  85. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/localize/__init__.py +0 -0
  86. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/localize/localize_torch.py +0 -0
  87. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/localize/localize_util.py +0 -0
  88. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/__init__.py +0 -0
  89. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/grab.py +0 -0
  90. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/matching_util/__init__.py +6 -6
  91. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  92. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/matching_util/drifty.py +0 -0
  93. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  94. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  95. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  96. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/shaver.py +0 -0
  97. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/peel/threshold.py +0 -0
  98. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  99. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/pretrained/detect_np1.pt +0 -0
  100. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/pretrained/detect_np2.pt +0 -0
  101. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  102. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  103. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  104. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  105. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  106. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  107. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/templates/__init__.py +0 -0
  108. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/templates/get_templates.py +0 -0
  109. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/templates/realignment.py +0 -0
  110. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/templates/templib.py +0 -0
  111. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/__init__.py +0 -0
  112. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  113. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/all_transformers.py +0 -0
  114. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/amortized_localization.py +0 -0
  115. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/decollider.py +0 -0
  116. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/denoising_scorer.py +0 -0
  117. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/enforce_decrease.py +0 -0
  118. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/fixed_prop.py +0 -0
  119. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/interp.py +0 -0
  120. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/localize.py +0 -0
  121. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/matching_denoiser.py +0 -0
  122. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  123. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/supervised_denoiser.py +0 -0
  124. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/temporal_pca.py +0 -0
  125. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/transform/whiten.py +0 -0
  126. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/__init__.py +0 -0
  127. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/drift_util.py +0 -0
  128. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/job_util.py +0 -0
  129. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/more_operators.py +0 -0
  130. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/nn_util.py +0 -0
  131. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/preprocess_util.py +0 -0
  132. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/registration_util.py +0 -0
  133. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/residual_util.py +0 -0
  134. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/sparse_util.py +0 -0
  135. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/spikeio.py +0 -0
  136. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/spiketorch.py +0 -0
  137. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/testing_util/__init__.py +0 -0
  138. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  139. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  140. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/torch_optimization_util.py +0 -0
  141. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/util/waveform_util.py +0 -0
  142. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/__init__.py +0 -0
  143. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/analysis_plots.py +0 -0
  144. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/colors.py +0 -0
  145. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/layout.py +0 -0
  146. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/metrics_util.py +0 -0
  147. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/mixture.py +0 -0
  148. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/recanim.py +0 -0
  149. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/unit.py +0 -0
  150. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/versus.py +0 -0
  151. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/vismain.py +0 -0
  152. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort/vis/waveforms.py +0 -0
  153. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort.egg-info/dependency_links.txt +0 -0
  154. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort.egg-info/entry_points.txt +0 -0
  155. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort.egg-info/requires.txt +0 -0
  156. {dartsort-0.5.21 → dartsort-0.5.23}/src/dartsort.egg-info/top_level.txt +0 -0
  157. {dartsort-0.5.21 → dartsort-0.5.23}/tests/conftest.py +0 -0
  158. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_agglomerate.py +0 -0
  159. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_config.py +0 -0
  160. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_dartsort.py +0 -0
  161. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_density.py +0 -0
  162. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_detect.py +0 -0
  163. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_drift_util.py +0 -0
  164. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_drifty_matching.py +0 -0
  165. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_enforce_decrease.py +0 -0
  166. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_kmeans.py +0 -0
  167. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_linalg.py +0 -0
  168. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_localize_torch.py +0 -0
  169. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_mixture_classifier.py +0 -0
  170. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_modeling.py +0 -0
  171. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_spiketorch.py +0 -0
  172. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_subtract.py +0 -0
  173. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_templates.py +0 -0
  174. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_threshold.py +0 -0
  175. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_torch_optimization_util.py +0 -0
  176. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_transform.py +0 -0
  177. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_util.py +0 -0
  178. {dartsort-0.5.21 → dartsort-0.5.23}/tests/test_waveform_util.py +0 -0
  179. {dartsort-0.5.21 → dartsort-0.5.23}/zensical.toml +0 -0
@@ -15,6 +15,7 @@ figs/*
15
15
  *.sublime-workspace
16
16
  *.sublime-project
17
17
  .vscode/
18
+ *.bin
18
19
 
19
20
  # sphinx
20
21
  docs/source/_autosummary/
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.21
3
+ Version: 0.5.23
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -75,7 +75,7 @@ line-length = 88
75
75
 
76
76
  [tool.ruff.lint]
77
77
  extend-select = ["E", "F", "I", "N", "B", "SIM", "NPY", "RUF", "PTH", "PD", "FURB"]
78
- ignore = ["F722", "N812", "SIM102", "SIM108", "E501", "N806", "N803", "N815", "RUF005", "SIM300", "C408", "TRY401"]
78
+ ignore = ["F722", "N812", "SIM102", "SIM103", "SIM105", "SIM108", "SIM114", "E501", "N806", "N803", "N815", "RUF005", "SIM300", "C408", "TRY401"]
79
79
 
80
80
  [tool.ruff.lint.per-file-ignores]
81
81
  "**/__init__.py" = ["F401", "F403"]
@@ -96,5 +96,4 @@ replace-imports-with-any = ["hdbscan"]
96
96
 
97
97
  [tool.ty.rules]
98
98
  "unsupported-base" = "ignore"
99
- "invalid-key" = "ignore"
100
99
  "possibly-unresolved-reference" = "error"
@@ -76,7 +76,12 @@ from .util.logging_util import (
76
76
  get_logger,
77
77
  set_log_level,
78
78
  )
79
- from .util.motion import MotionInfo, get_motion_info, try_load_motion_info
79
+ from .util.motion import (
80
+ MotionInfo,
81
+ detect_for_motion,
82
+ get_motion_info,
83
+ try_load_motion_info,
84
+ )
80
85
  from .util.noise_util import EmbeddedNoise
81
86
  from .util.preprocess_util import preprocess
82
87
  from .util.py_util import databag, ensure_path
@@ -0,0 +1,4 @@
1
+ from .cli import dartsort_cli
2
+
3
+ if __name__ == "__main__":
4
+ dartsort_cli()
@@ -1,5 +1,6 @@
1
1
  import argparse
2
2
  import logging
3
+ import pickle
3
4
  import sys
4
5
 
5
6
  import spikeinterface.core as sc
@@ -13,9 +14,7 @@ def dartsort_cli():
13
14
 
14
15
  Try `dartsort --help` to start.
15
16
 
16
- ---<!!> Not stable.
17
-
18
- I am figuring out how to do preprocessing still. It may be configured?
17
+ ---
19
18
  """
20
19
  # -- define CLI
21
20
  assert dartsort_cli.__doc__ is not None
@@ -90,7 +89,12 @@ def dartsort_cli():
90
89
 
91
90
  # load the recording
92
91
  try:
93
- rec = sc.load(cli_util.ensurepath(args.recording))
92
+ rec_path = cli_util.ensurepath(args.recording, strict=True)
93
+ if rec_path.suffix == ".pkl":
94
+ with rec_path.open("rb") as jar:
95
+ rec = pickle.load(jar)
96
+ else:
97
+ rec = sc.load(rec_path)
94
98
  assert isinstance(rec, sc.BaseRecording)
95
99
  except FileNotFoundError as e:
96
100
  ee = FileNotFoundError(
@@ -11,7 +11,11 @@ from spikeinterface.core import BaseRecording
11
11
 
12
12
  from ..templates.template_util import shared_basis_compress_templates
13
13
  from ..templates.templates import TemplateData
14
- from ..util.data_util import DARTsortSorting
14
+ from ..util.data_util import (
15
+ DARTsortSorting,
16
+ count_not_sorted,
17
+ pos_int_unique_and_counts,
18
+ )
15
19
  from ..util.internal_config import (
16
20
  ComputationConfig,
17
21
  RefinementConfig,
@@ -72,7 +76,7 @@ def agglomerate(
72
76
 
73
77
  if template_data is None:
74
78
  did_flatten = True
75
- sorting = sorting.flatten(include_gmm_properties=True)
79
+ sorting = sorting.flatten(include_gmm_properties=True, in_place=True)
76
80
  else:
77
81
  did_flatten = False
78
82
 
@@ -108,7 +112,7 @@ def agglomerate(
108
112
  link=template_merge_cfg.linkage,
109
113
  )
110
114
  elif not did_flatten:
111
- agg_sorting = sorting.flatten(include_gmm_properties=True)
115
+ agg_sorting = sorting.flatten(include_gmm_properties=True, in_place=True)
112
116
  new_ids = None
113
117
  else:
114
118
  agg_sorting = sorting
@@ -117,7 +121,9 @@ def agglomerate(
117
121
  if refinement_cfg is not None:
118
122
  agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
119
123
 
120
- agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
124
+ agg_sorting, reorder = reorder_by_depth(
125
+ agg_sorting, motion=motion, in_place=True
126
+ )
121
127
  if new_ids is None:
122
128
  new_ids = reorder
123
129
  else:
@@ -241,7 +247,7 @@ def agglomerate(
241
247
 
242
248
  agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
243
249
 
244
- agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
250
+ agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion, in_place=True)
245
251
  new_ids = reorder[np.unique(new_ids, return_inverse=True)[1]]
246
252
 
247
253
  return Agglomeration(
@@ -862,7 +868,7 @@ def combine_gmm_scores(
862
868
  )
863
869
 
864
870
 
865
- @numba.njit(parallel=True)
871
+ @numba.njit(parallel=True, nogil=True)
866
872
  def _combine_loop(
867
873
  cand: np.ndarray,
868
874
  new_id_counts: np.ndarray,
@@ -932,7 +938,10 @@ def deduplicate_spikes(
932
938
  new_labels = sorting.labels.copy()
933
939
  scores = None
934
940
  for sck in score_by:
935
- scores = getattr(sorting, sck, None)
941
+ if not sorting.has_dataset(sck):
942
+ continue
943
+ sl = (slice(None), 0) if sck.endswith("log_liks") else ()
944
+ scores = sorting.load_dataset(sck, sl=sl)
936
945
  if scores is not None:
937
946
  logger.dartsortdebug(f"deduplicate by score {sck}")
938
947
  break
@@ -944,33 +953,38 @@ def deduplicate_spikes(
944
953
  assert scores.shape == new_labels.shape
945
954
 
946
955
  # handle unsorted times
947
- tsort = np.argsort(sorting.times_samples)
948
- new_labels = new_labels[tsort]
949
- times_samples = sorting.times_samples[tsort]
950
- scores = scores[tsort]
956
+ if count_not_sorted(sorting.times_samples) > 0:
957
+ tsort = np.argsort(sorting.times_samples, kind="stable")
958
+ new_labels = new_labels[tsort]
959
+ times_samples = sorting.times_samples[tsort]
960
+ scores = scores[tsort]
961
+ else:
962
+ times_samples = sorting.times_samples
963
+ tsort = None
951
964
 
952
- unit_ids = np.unique(new_labels)
953
- unit_ids = unit_ids[unit_ids >= 0]
965
+ unit_ids, _, _ = pos_int_unique_and_counts(new_labels)
954
966
  ndrop = 0
967
+ unit_mask_tmp = np.empty(new_labels.shape, dtype=bool)
955
968
  for unit_id in unit_ids:
956
- in_unit = np.flatnonzero(new_labels == unit_id)
969
+ in_unit = np.flatnonzero(np.equal(new_labels, unit_id, out=unit_mask_tmp))
957
970
  if in_unit.size <= 1:
958
971
  continue
959
972
  t = times_samples[in_unit]
960
973
  dt = np.diff(t)
961
974
  if dt.min() > radius_samples:
962
975
  continue
963
- discard = _dedup_unit(t, dt, scores[in_unit], radius_samples)
976
+ discard = dedup_unit(t, dt, scores[in_unit], radius_samples)
964
977
  ndrop += discard.sum()
965
978
  new_labels[in_unit[discard]] = -1
966
979
 
967
980
  logger.dartsortdebug(f"drop {ndrop}/{len(sorting)} isi violator spikes")
968
- new_labels = new_labels[np.argsort(tsort)]
981
+ if tsort is not None:
982
+ new_labels = new_labels[np.argsort(tsort)]
969
983
 
970
984
  return sorting.ephemeral_replace(labels=new_labels)
971
985
 
972
986
 
973
- def _dedup_unit(
987
+ def dedup_unit(
974
988
  t: np.ndarray, dt: np.ndarray, scores: np.ndarray, radius: int
975
989
  ) -> np.ndarray:
976
990
  """Deduplicate a single unit's spike train."""
@@ -979,7 +993,7 @@ def _dedup_unit(
979
993
  return discard
980
994
 
981
995
 
982
- @numba.njit
996
+ @numba.njit(nogil=True)
983
997
  def _dedup_unit_loop(
984
998
  dt: np.ndarray, scores: np.ndarray, radius: int, discard: np.ndarray
985
999
  ):
@@ -1129,7 +1143,7 @@ def coentropy(
1129
1143
  )
1130
1144
 
1131
1145
 
1132
- @numba.njit(parallel=True)
1146
+ @numba.njit(parallel=True, nogil=True)
1133
1147
  def _calc_coentropy(
1134
1148
  coentropy: np.ndarray,
1135
1149
  cooccurrence: np.ndarray,
@@ -1142,7 +1156,7 @@ def _calc_coentropy(
1142
1156
  u = cands[i]
1143
1157
  q = resps[i]
1144
1158
  log_q = np.log(q)
1145
- np.nan_to_num(log_q, copy=False, neginf=0.0)
1159
+ np.nan_to_num(log_q, copy=False, neginf=0.0, posinf=np.inf)
1146
1160
  dh = q * log_q
1147
1161
 
1148
1162
  ui0 = u[0]
@@ -1,14 +1,23 @@
1
1
  from typing import cast
2
2
 
3
3
  import h5py
4
+ import numba
4
5
  import numpy as np
5
6
  from scipy.cluster.hierarchy import fcluster, linkage
6
7
  from scipy.spatial import KDTree
7
8
 
8
- from ..util import data_util, waveform_util
9
- from ..util.data_util import DARTsortSorting
9
+ from ..util import waveform_util
10
+ from ..util.data_util import (
11
+ DARTsortSorting,
12
+ apply_label_remapping_in_place,
13
+ count_not_sorted,
14
+ mean_by_label_1d,
15
+ pos_int_unique_and_counts,
16
+ yield_masked_chunks,
17
+ )
10
18
  from ..util.logging_util import get_logger
11
19
  from ..util.motion import MotionInfo
20
+ from ..util.py_util import databag
12
21
 
13
22
  logger = get_logger(__name__)
14
23
 
@@ -59,9 +68,8 @@ def apply_reclustering(
59
68
  assert sorting.labels is not None
60
69
 
61
70
  if new_labels is None:
62
- new_labels = np.full_like(sorting.labels, -1)
63
- kept = np.flatnonzero(sorting.labels >= 0)
64
- new_labels[kept] = merge_mapping[sorting.labels[kept]]
71
+ new_labels = sorting.labels.copy()
72
+ apply_label_remapping_in_place(new_labels, merge_mapping)
65
73
 
66
74
  if shifts is None:
67
75
  return sorting.ephemeral_replace(labels=new_labels)
@@ -99,7 +107,6 @@ def hierarchical_cluster(
99
107
  threshold=1.0,
100
108
  eps=1e-5,
101
109
  ):
102
- """"""
103
110
  n = distances.shape[0]
104
111
  assert eps < threshold # that would be confusing.
105
112
  if n <= 1:
@@ -117,8 +124,8 @@ def hierarchical_cluster(
117
124
  if labels is None:
118
125
  return None, np.arange(n)
119
126
  else:
120
- ids = np.unique(labels)
121
- return labels, ids[ids >= 0]
127
+ ids, _, _ = pos_int_unique_and_counts(labels)
128
+ return labels, ids
122
129
 
123
130
  if not finite.all():
124
131
  inf = max(0, pdist[finite].max()) + threshold + 1.0
@@ -175,8 +182,7 @@ def sparsify_labels(
175
182
  ) -> dict[int, np.ndarray]:
176
183
  assert labels.ndim == 1
177
184
  if ids is None:
178
- ids = np.unique(labels)
179
- ids = ids[ids >= 0]
185
+ ids, _, _ = pos_int_unique_and_counts(labels)
180
186
  inj = {}
181
187
  for j in ids:
182
188
  inj[j] = np.flatnonzero(labels == j)
@@ -265,7 +271,7 @@ def maximal_leaf_groups(
265
271
 
266
272
  assert max(map(len, groups)) <= max_group_size
267
273
  assert sum(map(len, groups)) == n
268
- assert set(gv for g in groups for gv in g) == set(range(n))
274
+ assert {gv for g in groups for gv in g} == set(range(n))
269
275
 
270
276
  groups = [tuple(sorted(g)) for g in groups]
271
277
 
@@ -329,6 +335,7 @@ def reorder_by_depth(
329
335
  spatial_footprints: np.ndarray | None = None,
330
336
  geom: np.ndarray | None = None,
331
337
  centroids: np.ndarray | None = None,
338
+ in_place: bool = False,
332
339
  ) -> tuple[DARTsortSorting, np.ndarray]:
333
340
  """Reorder cluster labels so that centroid depth is increasing
334
341
 
@@ -346,11 +353,8 @@ def reorder_by_depth(
346
353
  reorder: np.ndarray
347
354
  reorder[j] is the new label of original unit j.
348
355
  """
356
+ sorting = sorting.flatten(include_gmm_properties=True, in_place=in_place)
349
357
  assert sorting.labels is not None
350
- kept = np.flatnonzero(sorting.labels >= 0)
351
- kept_labels = sorting.labels[kept]
352
-
353
- units, kept_labels = np.unique(kept_labels, return_inverse=True)
354
358
 
355
359
  if geom is None and motion is not None:
356
360
  geom = motion.rgeom
@@ -359,26 +363,19 @@ def reorder_by_depth(
359
363
  assert centroids is None
360
364
  assert geom is not None
361
365
  assert spatial_footprints.shape[1] == geom.shape[0]
362
- assert spatial_footprints.shape[0] == units.shape[0]
363
366
  w = spatial_footprints / spatial_footprints.sum(1, keepdims=True)
364
367
  assert np.isfinite(w).all()
365
368
  centroids = w @ geom[:, 1]
366
369
 
367
370
  if centroids is None:
368
- depths = sorting.point_source_localizations[kept, 2]
371
+ z = sorting.load_dataset("point_source_localizations", sl=(slice(None), 2))
369
372
  if motion is not None:
370
- depths = motion.correct_s(sorting.times_seconds[kept], depths)
371
-
372
- centroids = np.zeros(units.size)
373
- for u in range(units.size):
374
- inu = np.flatnonzero(kept_labels == u)
375
- centroids[u] = np.median(depths[inu])
376
- assert centroids.shape[0] == units.shape[0]
373
+ z = motion.correct_s(sorting.times_seconds, z)
374
+ centroids = mean_by_label_1d(sorting, z)
377
375
 
378
- labels = sorting.labels.copy()
379
- # this one is some food for thought, lol.
376
+ labels = sorting.labels if in_place else sorting.labels.copy()
380
377
  reorder = np.argsort(np.argsort(centroids, kind="stable"), kind="stable")
381
- labels[kept] = reorder[kept_labels]
378
+ apply_label_remapping_in_place(labels, reorder)
382
379
  reordered_sorting = sorting.ephemeral_replace(labels=labels)
383
380
 
384
381
  return reordered_sorting, reorder
@@ -516,7 +513,7 @@ def get_main_channel_pcs(
516
513
  with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
517
514
  feats_dset = h5[dataset_name]
518
515
  channel_index = cast(h5py.Dataset, h5["channel_index"])[:]
519
- for ixs, feats in data_util.yield_masked_chunks(
516
+ for ixs, feats in yield_masked_chunks(
520
517
  mask, feats_dset, show_progress=show_progress, desc_prefix="Main channel"
521
518
  ):
522
519
  feats = feats[:, :rank]
@@ -527,7 +524,7 @@ def get_main_channel_pcs(
527
524
  return features
528
525
 
529
526
 
530
- def decrumb(labels: np.ndarray, min_size: int = 5, in_place=False, flatten=True):
527
+ def decrumb_labels(labels: np.ndarray, min_size: int = 5, in_place=False, flatten=True):
531
528
  """Remove small units
532
529
 
533
530
  Parameters
@@ -543,27 +540,142 @@ def decrumb(labels: np.ndarray, min_size: int = 5, in_place=False, flatten=True)
543
540
  labels
544
541
  The (flattened) decrumbed labels.
545
542
  """
546
- kept = np.flatnonzero(labels >= 0)
547
- labels_kept = labels[kept]
548
- labels = labels if in_place else labels.copy()
549
-
550
- units_sparse, counts_sparse = np.unique(labels_kept, return_counts=True)
551
- if not units_sparse.size:
543
+ units, counts, _ = pos_int_unique_and_counts(labels)
544
+ if not units.size:
552
545
  return labels
546
+ all_big = counts.min() >= min_size
547
+ flat_ok = (not flatten) or np.array_equal(units, np.arange(len(units)))
548
+ if all_big and flat_ok:
549
+ return labels
550
+ remapping = np.full((units.max() + 1,), -1, dtype=labels.dtype)
551
+ kept_units = units[counts >= min_size]
552
+ if flatten:
553
+ remapping[kept_units] = np.arange(len(kept_units))
554
+ else:
555
+ remapping[kept_units] = kept_units
556
+ new_labels = labels if in_place else labels.copy()
557
+ apply_label_remapping_in_place(new_labels, remapping)
558
+ logger.dartsortdebug(f"decrumb ({min_size}): {units.size}->{kept_units.size}.")
559
+ return new_labels
553
560
 
554
- k = units_sparse.max() + 1
555
- counts = np.zeros(k, dtype=counts_sparse.dtype)
556
- counts[units_sparse] = counts_sparse
557
- units = np.arange(k)
558
561
 
559
- big_enough = counts >= min_size
560
- k1 = big_enough.sum()
561
- units[np.logical_not(big_enough)] = -1
562
+ def decrumb(
563
+ sorting: DARTsortSorting, min_size: int = 5, in_place=False, flatten=True
564
+ ) -> DARTsortSorting:
565
+ assert sorting.labels is not None
566
+ units, counts, _ = pos_int_unique_and_counts(sorting.labels)
567
+ if not units.size:
568
+ return sorting
569
+ all_big = counts.min() >= min_size
570
+ flat_ok = (not flatten) or np.array_equal(units, np.arange(len(units)))
571
+ if all_big and flat_ok:
572
+ return sorting
573
+
574
+ remapping = np.full((units.max() + 1,), -1, dtype=sorting.labels.dtype)
575
+ kept_units = units[counts >= min_size]
576
+ remapping[kept_units] = kept_units
577
+ new_labels = sorting.labels if in_place else sorting.labels.copy()
578
+ apply_label_remapping_in_place(new_labels, remapping)
579
+ sorting = sorting.ephemeral_replace(labels=new_labels)
562
580
  if flatten:
563
- units[big_enough] = np.arange(k1)
581
+ sorting = sorting.flatten(in_place=in_place)
582
+ return sorting
564
583
 
565
- logger.dartsortdebug(f"decrumb ({min_size}): {k}->{k1}.")
566
584
 
567
- labels[kept] = units[labels_kept]
585
+ @databag
586
+ class ViolationCounts:
587
+ unit_ids: np.ndarray
588
+ spike_counts: np.ndarray
589
+ """Same shape as unit_ids (flat)"""
590
+ viol_counts: np.ndarray
591
+ """Indexed by pair of ids (not flat)"""
568
592
 
569
- return labels
593
+
594
+ def violation_matrix(
595
+ st: DARTsortSorting, *, censor_ms: float = 0.25, viol_ms: float = 1.0
596
+ ) -> ViolationCounts:
597
+ """Count ACG and CCG violations within viol_ms
598
+
599
+ Times within censor_ms of each other are ignored in the violation
600
+ count. The censorship is right-exclusive, so that if censor_ms is 0,
601
+ exact duplicates are counted; if censor_ms corresponds to 10 samples,
602
+ 9-sample viols are excluded and 10-sample viols are counted.
603
+ """
604
+ assert st.labels is not None
605
+ censor_samples = int(censor_ms * (st.sampling_frequency / 1000.0))
606
+ viol_samples = int(viol_ms * (st.sampling_frequency / 1000.0))
607
+
608
+ unit_ids, spike_counts, _ = pos_int_unique_and_counts(st.labels)
609
+ nu = (unit_ids.max() + 1).item() if unit_ids.size else 0
610
+ if not nu or (viol_samples < max(0, censor_samples)):
611
+ # nothing can be counted, but keep the matrix shape consistent
612
+ return ViolationCounts(
613
+ unit_ids=unit_ids,
614
+ spike_counts=spike_counts,
615
+ viol_counts=np.zeros((nu, nu), dtype=np.int64),
616
+ )
617
+
618
+ labels = st.labels
619
+ times = st.times_samples
620
+ if count_not_sorted(times) > 0:
621
+ tsort = np.argsort(times, kind="stable")
622
+ labels = labels[tsort]
623
+ times = times[tsort]
624
+
625
+ # count in chunks, per thread buffer; counts are ti<=tj
626
+ n = times.size
627
+ nchunks = max(1, numba.get_num_threads())
628
+ nchunks = min(nchunks, max(n, 1))
629
+ starts = (np.arange(nchunks + 1) * n) // nchunks
630
+ viol_counts = np.zeros((nchunks, nu, nu), dtype=np.int64)
631
+
632
+ _violation_count_matrix(
633
+ times, labels, censor_samples, viol_samples, starts, viol_counts
634
+ )
635
+
636
+ viol_counts = viol_counts.sum(axis=0)
637
+ viol_diag = np.diagonal(viol_counts).copy()
638
+ viol_counts += viol_counts.T
639
+ np.fill_diagonal(viol_counts, viol_diag)
640
+
641
+ return ViolationCounts(
642
+ unit_ids=unit_ids,
643
+ spike_counts=spike_counts,
644
+ viol_counts=viol_counts,
645
+ )
646
+
647
+
648
+ @numba.njit(nogil=True, parallel=True)
649
+ def _violation_count_matrix(
650
+ times: np.ndarray,
651
+ labels: np.ndarray,
652
+ censor_samples: int,
653
+ viol_samples: int,
654
+ starts: np.ndarray,
655
+ counts: np.ndarray,
656
+ ):
657
+ n = times.shape[0]
658
+
659
+ # parallelize over chunks
660
+ for c in numba.prange(starts.shape[0] - 1): # ty: ignore[not-iterable]
661
+ out = counts[c]
662
+
663
+ for i in range(starts[c], starts[c + 1]):
664
+ li = labels[i]
665
+ if li < 0:
666
+ continue
667
+
668
+ ti = times[i]
669
+ first = ti + censor_samples
670
+ last = ti + viol_samples
671
+
672
+ # make sure to read js past the chunk end!
673
+ for j in range(i + 1, n):
674
+ if times[j] < first:
675
+ continue
676
+ if times[j] > last: # be inclusive here i suppose
677
+ break
678
+ lj = labels[j]
679
+ if lj < 0:
680
+ continue
681
+ out[li, lj] += 1
@@ -1,5 +1,6 @@
1
1
  import gc
2
- from typing import TYPE_CHECKING, Literal, Self, Sequence, cast
2
+ from collections.abc import Sequence
3
+ from typing import TYPE_CHECKING, Literal, Self, cast
3
4
 
4
5
  import numpy as np
5
6
  import sklearn.cluster
@@ -473,7 +474,7 @@ class DensityPeaksClusterer(Clusterer):
473
474
  del kres
474
475
  cleanup_and_log_gpu_usage(self.computation_cfg, "DPC->kmeans")
475
476
 
476
- labels = cluster_util.decrumb(
477
+ labels = cluster_util.decrumb_labels(
477
478
  labels, min_size=self.remove_clusters_smaller_than, in_place=True
478
479
  )
479
480