DARTsort 0.5.20b0__tar.gz → 0.5.22__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. {dartsort-0.5.20b0 → dartsort-0.5.22}/.gitignore +1 -0
  2. {dartsort-0.5.20b0 → dartsort-0.5.22}/PKG-INFO +1 -1
  3. {dartsort-0.5.20b0 → dartsort-0.5.22}/pyproject.toml +1 -1
  4. dartsort-0.5.22/src/dartsort/__main__.py +4 -0
  5. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/cli.py +9 -5
  6. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/clustering/agglomerate.py +33 -19
  7. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/clustering/cluster_util.py +56 -47
  8. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/clustering/clustering.py +3 -2
  9. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/clustering/clustering_features.py +82 -103
  10. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/clustering/density.py +4 -4
  11. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/clustering/mixture.py +4 -10
  12. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/clustering/refine_util.py +1 -4
  13. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/config.py +1 -1
  14. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/evaluate/analysis.py +2 -2
  15. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/evaluate/comparison.py +14 -14
  16. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/evaluate/noise_recording_tools.py +1 -1
  17. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/evaluate/sim_template_tools.py +179 -84
  18. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/main.py +45 -14
  19. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/matching.py +0 -6
  20. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/templates/postprocess_util.py +8 -8
  21. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/templates/templates.py +0 -2
  22. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/amplitudes.py +3 -0
  23. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/mixture_classifier.py +2 -1
  24. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/pipeline.py +6 -5
  25. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/reduction.py +3 -6
  26. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/transform_base.py +5 -2
  27. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/cli_util.py +2 -2
  28. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/data_util.py +309 -69
  29. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/internal_config.py +8 -4
  30. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/interpolation_util.py +18 -14
  31. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/main_util.py +9 -2
  32. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/motion.py +11 -3
  33. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/multiprocessing_util.py +1 -1
  34. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/noise_util.py +2 -2
  35. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/peel_util.py +11 -15
  36. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/spiketorch.py +1 -2
  37. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort.egg-info/PKG-INFO +1 -1
  38. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort.egg-info/SOURCES.txt +1 -0
  39. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_alignment.py +3 -4
  40. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_cluster_util.py +2 -2
  41. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_clustering.py +3 -2
  42. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_matching.py +1 -1
  43. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_simkit.py +92 -17
  44. {dartsort-0.5.20b0 → dartsort-0.5.22}/.github/workflows/ci.yml +0 -0
  45. {dartsort-0.5.20b0 → dartsort-0.5.22}/.github/workflows/ci_windows.yml +0 -0
  46. {dartsort-0.5.20b0 → dartsort-0.5.22}/.github/workflows/deploy.yml +0 -0
  47. {dartsort-0.5.20b0 → dartsort-0.5.22}/.github/workflows/docs.yml +0 -0
  48. {dartsort-0.5.20b0 → dartsort-0.5.22}/README.md +0 -0
  49. {dartsort-0.5.20b0 → dartsort-0.5.22}/docs/index.md +0 -0
  50. {dartsort-0.5.20b0 → dartsort-0.5.22}/docs/main_api.md +0 -0
  51. {dartsort-0.5.20b0 → dartsort-0.5.22}/docs/toolkit_api.md +0 -0
  52. {dartsort-0.5.20b0 → dartsort-0.5.22}/environment.yml +0 -0
  53. {dartsort-0.5.20b0 → dartsort-0.5.22}/notebook/.keep +0 -0
  54. {dartsort-0.5.20b0 → dartsort-0.5.22}/notebook/ibl_dandi_example.py +0 -0
  55. {dartsort-0.5.20b0 → dartsort-0.5.22}/setup.cfg +0 -0
  56. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/__init__.py +0 -0
  57. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/clustering/__init__.py +0 -0
  58. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/clustering/forward_backward.py +0 -0
  59. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/clustering/kmeans.py +0 -0
  60. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/clustering/merge.py +0 -0
  61. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/detect/__init__.py +0 -0
  62. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/detect/detect.py +0 -0
  63. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/evaluate/__init__.py +0 -0
  64. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/evaluate/hybrid_util.py +0 -0
  65. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/evaluate/simkit.py +0 -0
  66. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/evaluate/simlib.py +0 -0
  67. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/localize/__init__.py +0 -0
  68. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/localize/localize_torch.py +0 -0
  69. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/localize/localize_util.py +0 -0
  70. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/__init__.py +0 -0
  71. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/grab.py +0 -0
  72. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/matching_util/__init__.py +0 -0
  73. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  74. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/matching_util/drifty.py +0 -0
  75. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  76. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  77. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  78. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/peel_base.py +0 -0
  79. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/peel_lib.py +0 -0
  80. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/reduction_template.py +0 -0
  81. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/shaver.py +0 -0
  82. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/subtract.py +0 -0
  83. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/peel/threshold.py +0 -0
  84. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  85. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/pretrained/detect_np1.pt +0 -0
  86. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/pretrained/detect_np2.pt +0 -0
  87. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  88. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  89. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  90. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  91. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  92. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  93. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/templates/__init__.py +0 -0
  94. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/templates/get_templates.py +0 -0
  95. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/templates/realignment.py +0 -0
  96. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/templates/template_util.py +0 -0
  97. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/templates/templib.py +0 -0
  98. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/__init__.py +0 -0
  99. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  100. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/all_transformers.py +0 -0
  101. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/amortized_localization.py +0 -0
  102. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/decollider.py +0 -0
  103. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/denoising_scorer.py +0 -0
  104. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/enforce_decrease.py +0 -0
  105. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/fixed_prop.py +0 -0
  106. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/interp.py +0 -0
  107. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/localize.py +0 -0
  108. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/matching_denoiser.py +0 -0
  109. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  110. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/supervised_denoiser.py +0 -0
  111. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/temporal_pca.py +0 -0
  112. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/transform/whiten.py +0 -0
  113. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/__init__.py +0 -0
  114. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/drift_util.py +0 -0
  115. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/job_util.py +0 -0
  116. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/logging_util.py +0 -0
  117. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/more_operators.py +0 -0
  118. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/nn_util.py +0 -0
  119. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/preprocess_util.py +0 -0
  120. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/py_util.py +0 -0
  121. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/registration_util.py +0 -0
  122. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/residual_util.py +0 -0
  123. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/sparse_util.py +0 -0
  124. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/spikeio.py +0 -0
  125. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/testing_util/__init__.py +0 -0
  126. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  127. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  128. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/torch_optimization_util.py +0 -0
  129. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/torch_util.py +0 -0
  130. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/util/waveform_util.py +0 -0
  131. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/__init__.py +0 -0
  132. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/analysis_plots.py +0 -0
  133. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/colors.py +0 -0
  134. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/gt.py +0 -0
  135. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/layout.py +0 -0
  136. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/metrics_util.py +0 -0
  137. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/mixture.py +0 -0
  138. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/recanim.py +0 -0
  139. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/scatterplots.py +0 -0
  140. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/sorting.py +0 -0
  141. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/unit.py +0 -0
  142. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/unit_comparison.py +0 -0
  143. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/versus.py +0 -0
  144. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/vismain.py +0 -0
  145. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort/vis/waveforms.py +0 -0
  146. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort.egg-info/dependency_links.txt +0 -0
  147. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort.egg-info/entry_points.txt +0 -0
  148. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort.egg-info/requires.txt +0 -0
  149. {dartsort-0.5.20b0 → dartsort-0.5.22}/src/dartsort.egg-info/top_level.txt +0 -0
  150. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/conftest.py +0 -0
  151. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_agglomerate.py +0 -0
  152. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_config.py +0 -0
  153. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_dartsort.py +0 -0
  154. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_data_util.py +0 -0
  155. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_density.py +0 -0
  156. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_detect.py +0 -0
  157. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_drift_util.py +0 -0
  158. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_drifty_matching.py +0 -0
  159. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_enforce_decrease.py +0 -0
  160. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_grab_and_featurize.py +0 -0
  161. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_kmeans.py +0 -0
  162. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_linalg.py +0 -0
  163. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_localize_torch.py +0 -0
  164. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_mixture_classifier.py +0 -0
  165. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_modeling.py +0 -0
  166. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_spiketorch.py +0 -0
  167. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_subtract.py +0 -0
  168. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_templates.py +0 -0
  169. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_threshold.py +0 -0
  170. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_torch_optimization_util.py +0 -0
  171. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_transform.py +0 -0
  172. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_util.py +0 -0
  173. {dartsort-0.5.20b0 → dartsort-0.5.22}/tests/test_waveform_util.py +0 -0
  174. {dartsort-0.5.20b0 → dartsort-0.5.22}/zensical.toml +0 -0
@@ -15,6 +15,7 @@ figs/*
15
15
  *.sublime-workspace
16
16
  *.sublime-project
17
17
  .vscode/
18
+ *.bin
18
19
 
19
20
  # sphinx
20
21
  docs/source/_autosummary/
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.20b0
3
+ Version: 0.5.22
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -75,7 +75,7 @@ line-length = 88
75
75
 
76
76
  [tool.ruff.lint]
77
77
  extend-select = ["E", "F", "I", "N", "B", "SIM", "NPY", "RUF", "PTH", "PD", "FURB"]
78
- ignore = ["F722", "N812", "SIM102", "SIM108", "E501", "N806", "N803", "N815", "RUF005", "SIM300", "C408", "TRY401"]
78
+ ignore = ["F722", "N812", "SIM102", "SIM103", "SIM105", "SIM108", "E501", "N806", "N803", "N815", "RUF005", "SIM300", "C408", "TRY401"]
79
79
 
80
80
  [tool.ruff.lint.per-file-ignores]
81
81
  "**/__init__.py" = ["F401", "F403"]
@@ -0,0 +1,4 @@
1
+ from .cli import dartsort_cli
2
+
3
+ if __name__ == "__main__":
4
+ dartsort_cli()
@@ -1,5 +1,6 @@
1
1
  import argparse
2
2
  import logging
3
+ import pickle
3
4
  import sys
4
5
 
5
6
  import spikeinterface.core as sc
@@ -13,9 +14,7 @@ def dartsort_cli():
13
14
 
14
15
  Try `dartsort --help` to start.
15
16
 
16
- ---<!!> Not stable.
17
-
18
- I am figuring out how to do preprocessing still. It may be configured?
17
+ ---
19
18
  """
20
19
  # -- define CLI
21
20
  assert dartsort_cli.__doc__ is not None
@@ -90,7 +89,12 @@ def dartsort_cli():
90
89
 
91
90
  # load the recording
92
91
  try:
93
- rec = sc.load(cli_util.ensurepath(args.recording))
92
+ rec_path = cli_util.ensurepath(args.recording, strict=True)
93
+ if rec_path.suffix == ".pkl":
94
+ with rec_path.open("rb") as jar:
95
+ rec = pickle.load(jar)
96
+ else:
97
+ rec = sc.load(rec_path)
94
98
  assert isinstance(rec, sc.BaseRecording)
95
99
  except FileNotFoundError as e:
96
100
  ee = FileNotFoundError(
@@ -105,4 +109,4 @@ def dartsort_cli():
105
109
 
106
110
  # -- run
107
111
  # TODO: maybe this script should dump to Phy?
108
- ret = main.dartsort(rec, output_dir, cfg=cfg, overwrite=args.overwrite)
112
+ main.dartsort(rec, output_dir, cfg=cfg, overwrite=args.overwrite)
@@ -11,7 +11,11 @@ from spikeinterface.core import BaseRecording
11
11
 
12
12
  from ..templates.template_util import shared_basis_compress_templates
13
13
  from ..templates.templates import TemplateData
14
- from ..util.data_util import DARTsortSorting
14
+ from ..util.data_util import (
15
+ DARTsortSorting,
16
+ count_not_sorted,
17
+ pos_int_unique_and_counts,
18
+ )
15
19
  from ..util.internal_config import (
16
20
  ComputationConfig,
17
21
  RefinementConfig,
@@ -72,7 +76,7 @@ def agglomerate(
72
76
 
73
77
  if template_data is None:
74
78
  did_flatten = True
75
- sorting = sorting.flatten(include_gmm_properties=True)
79
+ sorting = sorting.flatten(include_gmm_properties=True, in_place=True)
76
80
  else:
77
81
  did_flatten = False
78
82
 
@@ -108,7 +112,7 @@ def agglomerate(
108
112
  link=template_merge_cfg.linkage,
109
113
  )
110
114
  elif not did_flatten:
111
- agg_sorting = sorting.flatten(include_gmm_properties=True)
115
+ agg_sorting = sorting.flatten(include_gmm_properties=True, in_place=True)
112
116
  new_ids = None
113
117
  else:
114
118
  agg_sorting = sorting
@@ -117,7 +121,9 @@ def agglomerate(
117
121
  if refinement_cfg is not None:
118
122
  agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
119
123
 
120
- agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
124
+ agg_sorting, reorder = reorder_by_depth(
125
+ agg_sorting, motion=motion, in_place=True
126
+ )
121
127
  if new_ids is None:
122
128
  new_ids = reorder
123
129
  else:
@@ -241,7 +247,7 @@ def agglomerate(
241
247
 
242
248
  agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
243
249
 
244
- agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
250
+ agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion, in_place=True)
245
251
  new_ids = reorder[np.unique(new_ids, return_inverse=True)[1]]
246
252
 
247
253
  return Agglomeration(
@@ -862,7 +868,7 @@ def combine_gmm_scores(
862
868
  )
863
869
 
864
870
 
865
- @numba.njit(parallel=True)
871
+ @numba.njit(parallel=True, nogil=True)
866
872
  def _combine_loop(
867
873
  cand: np.ndarray,
868
874
  new_id_counts: np.ndarray,
@@ -932,7 +938,10 @@ def deduplicate_spikes(
932
938
  new_labels = sorting.labels.copy()
933
939
  scores = None
934
940
  for sck in score_by:
935
- scores = getattr(sorting, sck, None)
941
+ if not sorting.has_dataset(sck):
942
+ continue
943
+ sl = (slice(None), 0) if sck.endswith("log_liks") else ()
944
+ scores = sorting.load_dataset(sck, sl=sl)
936
945
  if scores is not None:
937
946
  logger.dartsortdebug(f"deduplicate by score {sck}")
938
947
  break
@@ -944,33 +953,38 @@ def deduplicate_spikes(
944
953
  assert scores.shape == new_labels.shape
945
954
 
946
955
  # handle unsorted times
947
- tsort = np.argsort(sorting.times_samples)
948
- new_labels = new_labels[tsort]
949
- times_samples = sorting.times_samples[tsort]
950
- scores = scores[tsort]
956
+ if count_not_sorted(sorting.times_samples) > 0:
957
+ tsort = np.argsort(sorting.times_samples, kind="stable")
958
+ new_labels = new_labels[tsort]
959
+ times_samples = sorting.times_samples[tsort]
960
+ scores = scores[tsort]
961
+ else:
962
+ times_samples = sorting.times_samples
963
+ tsort = None
951
964
 
952
- unit_ids = np.unique(new_labels)
953
- unit_ids = unit_ids[unit_ids >= 0]
965
+ unit_ids, _, _ = pos_int_unique_and_counts(new_labels)
954
966
  ndrop = 0
967
+ unit_mask_tmp = np.empty(new_labels.shape, dtype=bool)
955
968
  for unit_id in unit_ids:
956
- in_unit = np.flatnonzero(new_labels == unit_id)
969
+ in_unit = np.flatnonzero(np.equal(new_labels, unit_id, out=unit_mask_tmp))
957
970
  if in_unit.size <= 1:
958
971
  continue
959
972
  t = times_samples[in_unit]
960
973
  dt = np.diff(t)
961
974
  if dt.min() > radius_samples:
962
975
  continue
963
- discard = _dedup_unit(t, dt, scores[in_unit], radius_samples)
976
+ discard = dedup_unit(t, dt, scores[in_unit], radius_samples)
964
977
  ndrop += discard.sum()
965
978
  new_labels[in_unit[discard]] = -1
966
979
 
967
980
  logger.dartsortdebug(f"drop {ndrop}/{len(sorting)} isi violator spikes")
968
- new_labels = new_labels[np.argsort(tsort)]
981
+ if tsort is not None:
982
+ new_labels = new_labels[np.argsort(tsort)]
969
983
 
970
984
  return sorting.ephemeral_replace(labels=new_labels)
971
985
 
972
986
 
973
- def _dedup_unit(
987
+ def dedup_unit(
974
988
  t: np.ndarray, dt: np.ndarray, scores: np.ndarray, radius: int
975
989
  ) -> np.ndarray:
976
990
  """Deduplicate a single unit's spike train."""
@@ -979,7 +993,7 @@ def _dedup_unit(
979
993
  return discard
980
994
 
981
995
 
982
- @numba.njit
996
+ @numba.njit(nogil=True)
983
997
  def _dedup_unit_loop(
984
998
  dt: np.ndarray, scores: np.ndarray, radius: int, discard: np.ndarray
985
999
  ):
@@ -1129,7 +1143,7 @@ def coentropy(
1129
1143
  )
1130
1144
 
1131
1145
 
1132
- @numba.njit(parallel=True)
1146
+ @numba.njit(parallel=True, nogil=True)
1133
1147
  def _calc_coentropy(
1134
1148
  coentropy: np.ndarray,
1135
1149
  cooccurrence: np.ndarray,
@@ -6,7 +6,12 @@ from scipy.cluster.hierarchy import fcluster, linkage
6
6
  from scipy.spatial import KDTree
7
7
 
8
8
  from ..util import data_util, waveform_util
9
- from ..util.data_util import DARTsortSorting
9
+ from ..util.data_util import (
10
+ DARTsortSorting,
11
+ apply_label_remapping_in_place,
12
+ mean_by_label_1d,
13
+ pos_int_unique_and_counts,
14
+ )
10
15
  from ..util.logging_util import get_logger
11
16
  from ..util.motion import MotionInfo
12
17
 
@@ -59,9 +64,8 @@ def apply_reclustering(
59
64
  assert sorting.labels is not None
60
65
 
61
66
  if new_labels is None:
62
- new_labels = np.full_like(sorting.labels, -1)
63
- kept = np.flatnonzero(sorting.labels >= 0)
64
- new_labels[kept] = merge_mapping[sorting.labels[kept]]
67
+ new_labels = sorting.labels.copy()
68
+ apply_label_remapping_in_place(new_labels, merge_mapping)
65
69
 
66
70
  if shifts is None:
67
71
  return sorting.ephemeral_replace(labels=new_labels)
@@ -99,7 +103,6 @@ def hierarchical_cluster(
99
103
  threshold=1.0,
100
104
  eps=1e-5,
101
105
  ):
102
- """"""
103
106
  n = distances.shape[0]
104
107
  assert eps < threshold # that would be confusing.
105
108
  if n <= 1:
@@ -117,8 +120,8 @@ def hierarchical_cluster(
117
120
  if labels is None:
118
121
  return None, np.arange(n)
119
122
  else:
120
- ids = np.unique(labels)
121
- return labels, ids[ids >= 0]
123
+ ids, _, _ = pos_int_unique_and_counts(labels)
124
+ return labels, ids
122
125
 
123
126
  if not finite.all():
124
127
  inf = max(0, pdist[finite].max()) + threshold + 1.0
@@ -175,8 +178,7 @@ def sparsify_labels(
175
178
  ) -> dict[int, np.ndarray]:
176
179
  assert labels.ndim == 1
177
180
  if ids is None:
178
- ids = np.unique(labels)
179
- ids = ids[ids >= 0]
181
+ ids, _, _ = pos_int_unique_and_counts(labels)
180
182
  inj = {}
181
183
  for j in ids:
182
184
  inj[j] = np.flatnonzero(labels == j)
@@ -265,7 +267,7 @@ def maximal_leaf_groups(
265
267
 
266
268
  assert max(map(len, groups)) <= max_group_size
267
269
  assert sum(map(len, groups)) == n
268
- assert set(gv for g in groups for gv in g) == set(range(n))
270
+ assert {gv for g in groups for gv in g} == set(range(n))
269
271
 
270
272
  groups = [tuple(sorted(g)) for g in groups]
271
273
 
@@ -329,6 +331,7 @@ def reorder_by_depth(
329
331
  spatial_footprints: np.ndarray | None = None,
330
332
  geom: np.ndarray | None = None,
331
333
  centroids: np.ndarray | None = None,
334
+ in_place: bool = False,
332
335
  ) -> tuple[DARTsortSorting, np.ndarray]:
333
336
  """Reorder cluster labels so that centroid depth is increasing
334
337
 
@@ -346,11 +349,8 @@ def reorder_by_depth(
346
349
  reorder: np.ndarray
347
350
  reorder[j] is the new label of original unit j.
348
351
  """
352
+ sorting = sorting.flatten(include_gmm_properties=True, in_place=in_place)
349
353
  assert sorting.labels is not None
350
- kept = np.flatnonzero(sorting.labels >= 0)
351
- kept_labels = sorting.labels[kept]
352
-
353
- units, kept_labels = np.unique(kept_labels, return_inverse=True)
354
354
 
355
355
  if geom is None and motion is not None:
356
356
  geom = motion.rgeom
@@ -359,26 +359,19 @@ def reorder_by_depth(
359
359
  assert centroids is None
360
360
  assert geom is not None
361
361
  assert spatial_footprints.shape[1] == geom.shape[0]
362
- assert spatial_footprints.shape[0] == units.shape[0]
363
362
  w = spatial_footprints / spatial_footprints.sum(1, keepdims=True)
364
363
  assert np.isfinite(w).all()
365
364
  centroids = w @ geom[:, 1]
366
365
 
367
366
  if centroids is None:
368
- depths = sorting.point_source_localizations[kept, 2]
367
+ z = sorting.load_dataset("point_source_localizations", sl=(slice(None), 2))
369
368
  if motion is not None:
370
- depths = motion.correct_s(sorting.times_seconds[kept], depths)
371
-
372
- centroids = np.zeros(units.size)
373
- for u in range(units.size):
374
- inu = np.flatnonzero(kept_labels == u)
375
- centroids[u] = np.median(depths[inu])
376
- assert centroids.shape[0] == units.shape[0]
369
+ z = motion.correct_s(sorting.times_seconds, z)
370
+ centroids = mean_by_label_1d(sorting, z)
377
371
 
378
- labels = sorting.labels.copy()
379
- # this one is some food for thought, lol.
372
+ labels = sorting.labels if in_place else sorting.labels.copy()
380
373
  reorder = np.argsort(np.argsort(centroids, kind="stable"), kind="stable")
381
- labels[kept] = reorder[kept_labels]
374
+ apply_label_remapping_in_place(labels, reorder)
382
375
  reordered_sorting = sorting.ephemeral_replace(labels=labels)
383
376
 
384
377
  return reordered_sorting, reorder
@@ -527,7 +520,7 @@ def get_main_channel_pcs(
527
520
  return features
528
521
 
529
522
 
530
- def decrumb(labels: np.ndarray, min_size: int = 5, in_place=False, flatten=True):
523
+ def decrumb_labels(labels: np.ndarray, min_size: int = 5, in_place=False, flatten=True):
531
524
  """Remove small units
532
525
 
533
526
  Parameters
@@ -543,27 +536,43 @@ def decrumb(labels: np.ndarray, min_size: int = 5, in_place=False, flatten=True)
543
536
  labels
544
537
  The (flattened) decrumbed labels.
545
538
  """
546
- kept = np.flatnonzero(labels >= 0)
547
- labels_kept = labels[kept]
548
- labels = labels if in_place else labels.copy()
549
-
550
- units_sparse, counts_sparse = np.unique(labels_kept, return_counts=True)
551
- if not units_sparse.size:
539
+ units, counts, _ = pos_int_unique_and_counts(labels)
540
+ if not units.size:
552
541
  return labels
553
-
554
- k = units_sparse.max() + 1
555
- counts = np.zeros(k, dtype=counts_sparse.dtype)
556
- counts[units_sparse] = counts_sparse
557
- units = np.arange(k)
558
-
559
- big_enough = counts >= min_size
560
- k1 = big_enough.sum()
561
- units[np.logical_not(big_enough)] = -1
542
+ all_big = counts.min() >= min_size
543
+ flat_ok = (not flatten) or np.array_equal(units, np.arange(len(units)))
544
+ if all_big and flat_ok:
545
+ return labels
546
+ remapping = np.full((units.max() + 1,), -1, dtype=labels.dtype)
547
+ kept_units = units[counts >= min_size]
562
548
  if flatten:
563
- units[big_enough] = np.arange(k1)
564
-
565
- logger.dartsortdebug(f"decrumb ({min_size}): {k}->{k1}.")
549
+ remapping[kept_units] = np.arange(len(kept_units))
550
+ else:
551
+ remapping[kept_units] = kept_units
552
+ new_labels = labels if in_place else labels.copy()
553
+ apply_label_remapping_in_place(new_labels, remapping)
554
+ logger.dartsortdebug(f"decrumb ({min_size}): {units.size}->{kept_units.size}.")
555
+ return new_labels
566
556
 
567
- labels[kept] = units[labels_kept]
568
557
 
569
- return labels
558
+ def decrumb(
559
+ sorting: DARTsortSorting, min_size: int = 5, in_place=False, flatten=True
560
+ ) -> DARTsortSorting:
561
+ assert sorting.labels is not None
562
+ units, counts, _ = pos_int_unique_and_counts(sorting.labels)
563
+ if not units.size:
564
+ return sorting
565
+ all_big = counts.min() >= min_size
566
+ flat_ok = (not flatten) or np.array_equal(units, np.arange(len(units)))
567
+ if all_big and flat_ok:
568
+ return sorting
569
+
570
+ remapping = np.full((units.max() + 1,), -1, dtype=sorting.labels.dtype)
571
+ kept_units = units[counts >= min_size]
572
+ remapping[kept_units] = kept_units
573
+ new_labels = sorting.labels if in_place else sorting.labels.copy()
574
+ apply_label_remapping_in_place(new_labels, remapping)
575
+ sorting = sorting.ephemeral_replace(labels=new_labels)
576
+ if flatten:
577
+ sorting = sorting.flatten(in_place=in_place)
578
+ return sorting
@@ -1,5 +1,6 @@
1
1
  import gc
2
- from typing import TYPE_CHECKING, Literal, Self, Sequence, cast
2
+ from collections.abc import Sequence
3
+ from typing import TYPE_CHECKING, Literal, Self, cast
3
4
 
4
5
  import numpy as np
5
6
  import sklearn.cluster
@@ -473,7 +474,7 @@ class DensityPeaksClusterer(Clusterer):
473
474
  del kres
474
475
  cleanup_and_log_gpu_usage(self.computation_cfg, "DPC->kmeans")
475
476
 
476
- labels = cluster_util.decrumb(
477
+ labels = cluster_util.decrumb_labels(
477
478
  labels, min_size=self.remove_clusters_smaller_than, in_place=True
478
479
  )
479
480