DARTsort 0.5.31__tar.gz → 0.5.33__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. {dartsort-0.5.31 → dartsort-0.5.33}/.github/workflows/docs.yml +7 -5
  2. {dartsort-0.5.31 → dartsort-0.5.33}/PKG-INFO +1 -1
  3. {dartsort-0.5.31 → dartsort-0.5.33}/environment.yml +1 -1
  4. {dartsort-0.5.31 → dartsort-0.5.33}/notebook/fit_and_batch_matching.py +100 -32
  5. {dartsort-0.5.31 → dartsort-0.5.33}/notebook/ibl_dandi_example.py +16 -18
  6. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/__init__.py +2 -0
  7. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/agglomerate.py +17 -5
  8. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/cluster_util.py +15 -2
  9. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/clustering.py +3 -1
  10. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/clustering_features.py +22 -3
  11. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/kmeans.py +56 -3
  12. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/mixture.py +1215 -529
  13. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/refine_util.py +3 -3
  14. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/config.py +38 -15
  15. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/hybridkit.py +2 -1
  16. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/noise_recording_tools.py +2 -2
  17. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/simkit.py +2 -2
  18. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/simlib.py +2 -2
  19. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/main.py +26 -10
  20. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/drifty.py +13 -2
  21. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/peel_base.py +191 -102
  22. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/reduction_template.py +164 -4
  23. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/subtract.py +2 -2
  24. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/threshold.py +8 -1
  25. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/get_templates.py +2 -0
  26. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/postprocess_util.py +2 -1
  27. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/template_util.py +2 -2
  28. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/templates.py +4 -0
  29. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/amortized_localization.py +13 -2
  30. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/decollider.py +6 -1
  31. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/mixture_classifier.py +17 -16
  32. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/pipeline.py +85 -88
  33. dartsort-0.5.33/src/dartsort/transform/reduction.py +590 -0
  34. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/supervised_denoiser.py +15 -2
  35. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/temporal_pca.py +13 -6
  36. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/transform_base.py +23 -8
  37. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/vq_matched_filter.py +2 -0
  38. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/whiten.py +2 -0
  39. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/data_util.py +82 -17
  40. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/drift_util.py +60 -9
  41. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/internal_config.py +148 -41
  42. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/interpolation_util.py +55 -19
  43. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/main_util.py +8 -3
  44. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/motion.py +41 -31
  45. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/multiprocessing_util.py +12 -0
  46. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/nn_util.py +9 -6
  47. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/peel_util.py +7 -2
  48. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/preprocess_util.py +1 -1
  49. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/py_util.py +11 -0
  50. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/spiketorch.py +126 -20
  51. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/gt.py +1 -1
  52. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/mixture.py +1 -1
  53. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/PKG-INFO +1 -1
  54. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_config.py +6 -0
  55. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_dartsort.py +24 -0
  56. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_drift_util.py +36 -0
  57. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_drifty_matching.py +55 -0
  58. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_matching.py +3 -3
  59. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_mixture_classifier.py +24 -0
  60. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_modeling.py +128 -18
  61. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_templates.py +204 -1
  62. dartsort-0.5.31/src/dartsort/transform/reduction.py +0 -269
  63. {dartsort-0.5.31 → dartsort-0.5.33}/.github/workflows/ci.yml +0 -0
  64. {dartsort-0.5.31 → dartsort-0.5.33}/.github/workflows/ci_windows.yml +0 -0
  65. {dartsort-0.5.31 → dartsort-0.5.33}/.github/workflows/deploy.yml +0 -0
  66. {dartsort-0.5.31 → dartsort-0.5.33}/.gitignore +0 -0
  67. {dartsort-0.5.31 → dartsort-0.5.33}/README.md +0 -0
  68. {dartsort-0.5.31 → dartsort-0.5.33}/docs/index.md +0 -0
  69. {dartsort-0.5.31 → dartsort-0.5.33}/docs/main_api.md +0 -0
  70. {dartsort-0.5.31 → dartsort-0.5.33}/docs/toolkit_api.md +0 -0
  71. {dartsort-0.5.31 → dartsort-0.5.33}/notebook/.keep +0 -0
  72. {dartsort-0.5.31 → dartsort-0.5.33}/pyproject.toml +0 -0
  73. {dartsort-0.5.31 → dartsort-0.5.33}/setup.cfg +0 -0
  74. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/__main__.py +0 -0
  75. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/cli.py +0 -0
  76. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/__init__.py +0 -0
  77. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/density.py +0 -0
  78. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/forward_backward.py +0 -0
  79. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/merge.py +0 -0
  80. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/detect/__init__.py +0 -0
  81. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/detect/detect.py +0 -0
  82. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/__init__.py +0 -0
  83. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/analysis.py +0 -0
  84. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/comparison.py +0 -0
  85. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/hybrid_util.py +0 -0
  86. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  87. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/localize/__init__.py +0 -0
  88. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/localize/localize_torch.py +0 -0
  89. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/localize/localize_util.py +0 -0
  90. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/__init__.py +0 -0
  91. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/grab.py +0 -0
  92. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching.py +0 -0
  93. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/__init__.py +0 -0
  94. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  95. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  96. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  97. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  98. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/peel_lib.py +0 -0
  99. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/shaver.py +0 -0
  100. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/subtract_util.py +0 -0
  101. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  102. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/detect_np1.pt +0 -0
  103. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/detect_np2.pt +0 -0
  104. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  105. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  106. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  107. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  108. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  109. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  110. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/__init__.py +0 -0
  111. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/realignment.py +0 -0
  112. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/templib.py +0 -0
  113. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/__init__.py +0 -0
  114. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  115. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/all_transformers.py +0 -0
  116. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/amplitudes.py +0 -0
  117. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/denoising_scorer.py +0 -0
  118. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/enforce_decrease.py +0 -0
  119. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/fixed_prop.py +0 -0
  120. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/interp.py +0 -0
  121. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/localize.py +0 -0
  122. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/matched_filter_net.py +0 -0
  123. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/matching_denoiser.py +0 -0
  124. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  125. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/__init__.py +0 -0
  126. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/cli_util.py +0 -0
  127. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/job_util.py +0 -0
  128. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/logging_util.py +0 -0
  129. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/more_operators.py +0 -0
  130. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/noise_util.py +0 -0
  131. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/registration_util.py +0 -0
  132. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/residual_util.py +0 -0
  133. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/sparse_util.py +0 -0
  134. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/spikeio.py +0 -0
  135. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/testing_util/__init__.py +0 -0
  136. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  137. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  138. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/torch_optimization_util.py +0 -0
  139. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/torch_util.py +0 -0
  140. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/waveform_util.py +0 -0
  141. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/__init__.py +0 -0
  142. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/analysis_plots.py +0 -0
  143. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/colors.py +0 -0
  144. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/layout.py +0 -0
  145. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/metrics_util.py +0 -0
  146. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/recanim.py +0 -0
  147. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/scatterplots.py +0 -0
  148. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/sorting.py +0 -0
  149. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/unit.py +0 -0
  150. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/unit_comparison.py +0 -0
  151. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/versus.py +0 -0
  152. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/vismain.py +0 -0
  153. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/waveforms.py +0 -0
  154. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/SOURCES.txt +0 -0
  155. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/dependency_links.txt +0 -0
  156. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/entry_points.txt +0 -0
  157. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/requires.txt +0 -0
  158. {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/top_level.txt +0 -0
  159. {dartsort-0.5.31 → dartsort-0.5.33}/tests/conftest.py +0 -0
  160. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_agglomerate.py +0 -0
  161. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_alignment.py +0 -0
  162. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_cluster_util.py +0 -0
  163. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_clustering.py +0 -0
  164. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_clustering_features.py +0 -0
  165. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_data_util.py +0 -0
  166. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_density.py +0 -0
  167. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_detect.py +0 -0
  168. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_enforce_decrease.py +0 -0
  169. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_grab_and_featurize.py +0 -0
  170. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_hybridkit.py +0 -0
  171. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_kmeans.py +0 -0
  172. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_linalg.py +0 -0
  173. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_localize_torch.py +0 -0
  174. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_multiprocessing_util.py +0 -0
  175. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_simkit.py +0 -0
  176. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_spiketorch.py +0 -0
  177. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_subtract.py +0 -0
  178. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_threshold.py +0 -0
  179. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_torch_optimization_util.py +0 -0
  180. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_transform.py +0 -0
  181. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_util.py +0 -0
  182. {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_waveform_util.py +0 -0
  183. {dartsort-0.5.31 → dartsort-0.5.33}/zensical.toml +0 -0
@@ -29,16 +29,18 @@ jobs:
29
29
  # this is a "fine-grained token"
30
30
  token: ${{ secrets.DARTSORT_GITHUB_IO }}
31
31
 
32
- - name: Setup Python
33
- uses: actions/setup-python@v6
32
+ - name: Setup micromamba
33
+ uses: mamba-org/setup-micromamba@v3
34
34
  with:
35
- python-version: 3.x
36
- cache: "pip" # caching pip dependencies
35
+ environment-file: dartsort/environment.yml
36
+ environment-name: dartsort-docs
37
37
 
38
38
  - name: pip install stuff
39
- run: python3 -m pip install -e dartsort/[doc]
39
+ shell: bash -el {0}
40
+ run: python -m pip install -e "dartsort/[doc]"
40
41
 
41
42
  - name: Zensical build
43
+ shell: bash -el {0}
42
44
  run: pushd dartsort/ && zensical build --clean && popd
43
45
 
44
46
  - name: overwrite site
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.31
3
+ Version: 0.5.33
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -3,7 +3,7 @@ name: dartsort
3
3
  channels:
4
4
  - conda-forge
5
5
  dependencies:
6
- - python>=3.11
6
+ - python>=3.12
7
7
  - h5py
8
8
  - ipykernel
9
9
  - ipython
@@ -5,11 +5,11 @@
5
5
  # extension: .py
6
6
  # format_name: percent
7
7
  # format_version: '1.3'
8
- # jupytext_version: 1.19.5
8
+ # jupytext_version: 1.19.6
9
9
  # kernelspec:
10
- # display_name: Python [conda env:dart14]
10
+ # display_name: Python [conda env:a]
11
11
  # language: python
12
- # name: conda-env-dart14-py
12
+ # name: conda-env-a-py
13
13
  # ---
14
14
 
15
15
  # %% [markdown]
@@ -31,18 +31,16 @@
31
31
  # recording.
32
32
 
33
33
  # %%
34
- import matplotlib.pyplot as plt
35
-
36
- # %%
37
- # dandi-related
38
- from dandi.dandiapi import DandiAPIClient
39
- import fsspec
34
+ import dataclasses
40
35
 
41
- # %%
36
+ import matplotlib.pyplot as plt
42
37
  import numpy as np
43
38
  import spikeinterface.full as si
39
+ from dandi.dandiapi import DandiAPIClient
40
+
44
41
  import dartsort
45
42
  import dartsort.vis as dartvis
43
+
46
44
  dartvis.set_plt_style()
47
45
 
48
46
  # %% [markdown]
@@ -50,12 +48,10 @@ dartvis.set_plt_style()
50
48
 
51
49
  # %%
52
50
  # update this for your machine!
53
- experiment_path = dartsort.ensure_path(
54
- "~/scratch/ibl-dandi-example/", mkdir=True
55
- )
51
+ experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
56
52
 
57
53
  # %%
58
- ds_dir = experiment_path / "dartsort"
54
+ ds_dir = experiment_path / "dartsort-chunked"
59
55
 
60
56
  # %%
61
57
  # IBL BWM dandiset, example recording
@@ -93,7 +89,7 @@ else:
93
89
  print(rec0)
94
90
 
95
91
  # slice 10 minutes near the end
96
- #TODO better IBL sampling rate handling
92
+ # TODO better IBL sampling rate handling
97
93
  rec = rec0
98
94
  end_time = rec.get_end_time()
99
95
  start_frame = rec.time_to_sample_index(end_time - 30 * 60)
@@ -103,36 +99,109 @@ else:
103
99
  rec._sampling_frequency = 30_000
104
100
 
105
101
  rec = dartsort.preprocess(rec, strategy="ibllike")
106
- rec = rec.save_to_folder(preprocessing_path)
102
+ rec = rec.save(folder=preprocessing_path, format="binary")
107
103
 
108
104
  # %%
109
105
  plt.imshow(
110
106
  rec.get_traces(0, 10_000, 11_000).T,
111
- aspect='auto',
107
+ aspect="auto",
112
108
  vmin=-5,
113
109
  vmax=5,
114
- interpolation='nearest',
110
+ interpolation="nearest",
115
111
  )
116
- plt.colorbar(shrink=0.5, label='standardized voltage')
117
- plt.xlabel('time (samples')
118
- plt.ylabel('channels');
119
-
112
+ plt.colorbar(shrink=0.5, label="standardized voltage")
113
+ plt.xlabel("time (samples")
114
+ plt.ylabel("channels")
120
115
  # %% [markdown]
121
116
  # # Configuration
122
117
 
123
118
  # %%
124
119
  # use defaults, except for the pipeline change
125
- cfg = dartsort.DARTsortUserConfig(fit_matching_models_only=True)
120
+ cfg = dartsort.DARTsortUserConfig()
126
121
 
127
122
  # %%
128
123
  # process into an internal config object for later use
129
124
  internal_cfg = dartsort.to_internal_config(cfg)
130
125
 
131
126
  # %% [markdown]
132
- # # Fit models
127
+ # # Initial spike detection and motion tracking model fits
133
128
 
134
129
  # %%
135
- dartsort.dartsort(rec, ds_dir, cfg=cfg)
130
+ dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_motion_models_only=True))
131
+
132
+ # %% [markdown]
133
+ # # Chunked spike detection for motion tracking
134
+ #
135
+ # While dartsort itself only needs to detect spikes on a subsample of the recording
136
+ # in order to do clustering, motion tracking does require spikes from the whole
137
+ # recording. dartsort uses a cheap but good spike detection and localization pass for
138
+ # this, which is very fast, but users at the thousand hour scale still might want to
139
+ # parallelize it.
140
+ #
141
+ # This runs that cheap spike detection for motion tracking in 1-minute batches for demo
142
+ # purposes.
143
+ #
144
+ # Here, the `chunk_starts_samples` argument slices the recording for us.
145
+ # This is better in this case than using `.frame_slice` or `.time_slice` on the recording,
146
+ # since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
147
+ #
148
+ # If the initial detection already covered the whole recording, motion tracking uses its
149
+ # spikes and this step is skipped.
150
+
151
+ # %%
152
+ subtraction_sorting = dartsort.load(ds_dir / "subtraction.h5")
153
+ need_motion_peaks = cfg.do_motion_estimation and dartsort.motion_needs_peaks(
154
+ internal_cfg, rec, subtraction_sorting
155
+ )
156
+ need_motion_peaks
157
+
158
+ # %%
159
+ chunk_length_samples = internal_cfg.motion_estimation_cfg.threshold_cfg.chunk_length_samples
160
+ batch_length_samples = 60 * chunk_length_samples
161
+ duration_samples = rec.get_num_samples()
162
+
163
+ if need_motion_peaks:
164
+ for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, batch_length_samples)):
165
+ batch_end_samples = min(duration_samples, batch_start_samples + batch_length_samples)
166
+ res = dartsort.detect_for_motion(
167
+ recording=rec,
168
+ output_dir=ds_dir,
169
+ previous_sorting=subtraction_sorting,
170
+ hdf5_filename=f"motionthreshold_batch{batch_ix:05}.h5",
171
+ chunk_starts_samples=np.arange(batch_start_samples, batch_end_samples, chunk_length_samples),
172
+ motion_cfg=internal_cfg.motion_estimation_cfg,
173
+ computation_cfg=internal_cfg.motion_computation_cfg,
174
+ sampling_cfg=internal_cfg.peeler_sampling_cfg,
175
+ waveform_cfg=internal_cfg.waveform_cfg,
176
+ )
177
+ del res
178
+
179
+ # %% [markdown]
180
+ # Combine the output of those detections into one sorting, and check that together the
181
+ # batches covered the recording.
182
+
183
+ # %%
184
+ if need_motion_peaks and not (ds_dir / "motionthreshold.h5").exists():
185
+ motion_chunk_h5s = sorted(ds_dir.glob("motionthreshold_batch*.h5"))
186
+ dartsort.virtual_concatenate_sorting_h5s(motion_chunk_h5s, ds_dir / "motionthreshold.h5")
187
+ motion_sorting = dartsort.load(ds_dir / "motionthreshold.h5")
188
+ assert np.array_equal(
189
+ motion_sorting.load_dataset("chunk_starts_samples"),
190
+ np.arange(0, duration_samples, chunk_length_samples),
191
+ )
192
+ print(motion_sorting)
193
+
194
+ # %%
195
+ # !ls -lah {ds_dir}
196
+
197
+ # %% [markdown]
198
+ # ^ Notice all of the chunked motion files! But motionthreshold.h5 itself is tiny, because the concatenation is virtual.
199
+
200
+ # %% [markdown]
201
+ # # Resume: estimate motion, cluster, learn templates and the matching step classifier
202
+
203
+ # %%
204
+ dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_matching_models_only=True))
136
205
 
137
206
  # %%
138
207
  # !ls {ds_dir}
@@ -146,11 +215,8 @@ dartsort.dartsort(rec, ds_dir, cfg=cfg)
146
215
  # %% [markdown]
147
216
  # # Chunked template matching
148
217
  #
149
- # This runs template matching in 1-minute batches.
150
- #
151
- # Here, `chunk_starts_samples` slices the recording for us.
152
- # This might be better than using `.frame_slice` or `.time_slice` on the recording,
153
- # since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
218
+ # This runs template matching in 1-minute batches, with the same loop over chunks
219
+ # as the motion tracking's spike detection above.
154
220
  #
155
221
 
156
222
  # %%
@@ -186,7 +252,9 @@ for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, 60 * 3
186
252
  # which can be turned off for users who don't need them.
187
253
 
188
254
  # %%
189
- sorting = dartsort.concatenate_sortings([dartsort.load(h5) for h5 in sorted(ds_dir.glob("matching1_batch*.h5"))])
255
+ sorting = dartsort.concatenate_sortings(
256
+ [dartsort.load(h5) for h5 in sorted(ds_dir.glob("matching1_batch*.h5"))]
257
+ )
190
258
 
191
259
  # %%
192
260
  sorting = dartsort.cluster(
@@ -199,7 +267,7 @@ sorting = dartsort.cluster(
199
267
 
200
268
  # %%
201
269
  # if you want to keep this...
202
- sorting.save(ds_dir / 'dartsort_sorting.npz')
270
+ sorting.save(ds_dir / "dartsort_sorting.npz")
203
271
 
204
272
  # %% [markdown]
205
273
  # # Visualize
@@ -5,25 +5,26 @@
5
5
  # extension: .py
6
6
  # format_name: percent
7
7
  # format_version: '1.3'
8
- # jupytext_version: 1.19.3
8
+ # jupytext_version: 1.19.6
9
9
  # kernelspec:
10
- # display_name: Python [conda env:dart14]
10
+ # display_name: Python [conda env:a]
11
11
  # language: python
12
- # name: conda-env-dart14-py
12
+ # name: conda-env-a-py
13
13
  # ---
14
14
 
15
15
  # %%
16
16
  import matplotlib.pyplot as plt
17
17
 
18
+ # %%
19
+ import spikeinterface.full as si
20
+
18
21
  # %%
19
22
  # dandi-related
20
23
  from dandi.dandiapi import DandiAPIClient
21
- import fsspec
22
24
 
23
- # %%
24
- import spikeinterface.full as si
25
25
  import dartsort
26
26
  import dartsort.vis as dartvis
27
+
27
28
  dartvis.set_plt_style()
28
29
 
29
30
  # %% [markdown]
@@ -31,9 +32,7 @@ dartvis.set_plt_style()
31
32
 
32
33
  # %%
33
34
  # update this for your machine!
34
- experiment_path = dartsort.ensure_path(
35
- "~/scratch/ibl-dandi-example/", mkdir=True
36
- )
35
+ experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
37
36
 
38
37
  # %%
39
38
  # IBL BWM dandiset, example recording
@@ -71,7 +70,7 @@ else:
71
70
  print(rec0)
72
71
 
73
72
  # slice 10 minutes near the end
74
- #TODO better IBL sampling rate handling
73
+ # TODO better IBL sampling rate handling
75
74
  rec = rec0
76
75
  end_time = rec.get_end_time()
77
76
  start_frame = rec.time_to_sample_index(end_time - 30 * 60)
@@ -81,20 +80,19 @@ else:
81
80
  rec._sampling_frequency = 30_000
82
81
 
83
82
  rec = dartsort.preprocess(rec, strategy="ibllike")
84
- rec = rec.save_to_folder(preprocessing_path)
83
+ rec = rec.save(folder=preprocessing_path, format="binary")
85
84
 
86
85
  # %%
87
86
  plt.imshow(
88
87
  rec.get_traces(0, 10_000, 11_000).T,
89
- aspect='auto',
88
+ aspect="auto",
90
89
  vmin=-5,
91
90
  vmax=5,
92
- interpolation='nearest',
91
+ interpolation="nearest",
93
92
  )
94
- plt.colorbar(shrink=0.5, label='standardized voltage')
95
- plt.xlabel('time (samples')
96
- plt.ylabel('channels');
97
-
93
+ plt.colorbar(shrink=0.5, label="standardized voltage")
94
+ plt.xlabel("time (samples")
95
+ plt.ylabel("channels")
98
96
  # %% [markdown]
99
97
  # ## Run *dartsort*
100
98
 
@@ -106,6 +104,6 @@ ds_res = dartsort.dartsort(rec, ds_dir)
106
104
  # # Visualize
107
105
 
108
106
  # %%
109
- dartvis.scatter_spike_features(sorting=ds_res["sorting"]);
107
+ dartvis.scatter_spike_features(sorting=ds_res["sorting"])
110
108
 
111
109
  # %%
@@ -63,6 +63,7 @@ from .util.data_util import (
63
63
  sorting_from_spikeinterface,
64
64
  try_get_denoising_pipeline,
65
65
  try_get_model_dir,
66
+ virtual_concatenate_sorting_h5s,
66
67
  )
67
68
  from .util.drift_util import registered_geometry
68
69
  from .util.internal_config import *
@@ -80,6 +81,7 @@ from .util.logging_util import (
80
81
  get_logger,
81
82
  set_log_level,
82
83
  )
84
+ from .util.main_util import motion_needs_peaks
83
85
  from .util.motion import (
84
86
  MotionInfo,
85
87
  detect_for_motion,
@@ -561,6 +561,7 @@ def template_distances(
561
561
  sorting=sorting,
562
562
  motion=motion,
563
563
  radius=template_merge_cfg.weighted_dist_radius,
564
+ registered_geom=template_data.registered_geom,
564
565
  )
565
566
  spatial_weights = radial_weights(radial_counts)
566
567
  assert np.isfinite(spatial_weights).all()
@@ -876,23 +877,34 @@ def bimod_stats(h):
876
877
  return a, b
877
878
 
878
879
 
879
- def radial_spike_counts(sorting: DARTsortSorting, motion: MotionInfo, radius: float):
880
+ def radial_spike_counts(
881
+ sorting: DARTsortSorting,
882
+ motion: MotionInfo,
883
+ radius: float,
884
+ registered_geom: np.ndarray | None = None,
885
+ ):
880
886
  assert sorting.labels is not None
887
+ if registered_geom is None:
888
+ registered_geom = motion.rgeom
881
889
  kept = np.flatnonzero(sorting.labels >= 0)
882
890
 
883
891
  # which reg chans do the spikes land on?
884
892
  x, z = motion.geom[sorting.channels[kept]].T
885
893
  cc = closest_registered_channels(
886
- times_seconds=sorting.times_seconds[kept], x=x, z_abs=z, motion=motion
894
+ times_seconds=sorting.times_seconds[kept],
895
+ x=x,
896
+ z_abs=z,
897
+ motion=motion,
898
+ registered_geom=registered_geom,
887
899
  )
888
900
 
889
901
  # count by label
890
902
  ll = sorting.labels[kept]
891
- counts = np.zeros((ll.max() + 1, motion.rgeom.shape[0]), dtype=np.int64)
903
+ counts = np.zeros((ll.max() + 1, registered_geom.shape[0]), dtype=np.int64)
892
904
  np.add.at(counts, (ll, cc), 1)
893
905
 
894
906
  # get radial neighborhoods
895
- ci = make_channel_index(motion.rgeom, radius, to_torch=False)
907
+ ci = make_channel_index(registered_geom, radius, to_torch=False)
896
908
 
897
909
  # puff out with radial neighborhood and sum up the counts by label
898
910
  radial_counts = np.zeros(counts.shape)
@@ -901,7 +913,7 @@ def radial_spike_counts(sorting: DARTsortSorting, motion: MotionInfo, radius: fl
901
913
  ii = np.flatnonzero(row)
902
914
  for channel, value in zip(ii, row[ii], strict=True):
903
915
  cixs = ci[channel]
904
- cixs = cixs[cixs < motion.rgeom.shape[0]]
916
+ cixs = cixs[cixs < registered_geom.shape[0]]
905
917
  radial_counts[uu, cixs] += value
906
918
 
907
919
  return radial_counts
@@ -428,7 +428,13 @@ def reorder_by_depth(
428
428
 
429
429
 
430
430
  def closest_registered_channels(
431
- *, times_seconds, x, z_abs, z_reg=None, motion: MotionInfo
431
+ *,
432
+ times_seconds,
433
+ x,
434
+ z_abs,
435
+ z_reg=None,
436
+ motion: MotionInfo,
437
+ registered_geom: np.ndarray | None = None,
432
438
  ) -> np.ndarray:
433
439
  """Assign spikes to the drift-extended channel closest to their registered position"""
434
440
  if z_reg is None:
@@ -436,7 +442,14 @@ def closest_registered_channels(
436
442
  z_reg = motion.correct_s(times_seconds, z_abs)
437
443
  reg_pos = np.c_[x, z_reg]
438
444
 
439
- _, reg_channels = motion.rgeom_kdt.query(reg_pos)
445
+ if registered_geom is None or (
446
+ registered_geom.shape == motion.rgeom.shape
447
+ and np.allclose(registered_geom, motion.rgeom)
448
+ ):
449
+ kdt = motion.rgeom_kdt
450
+ else:
451
+ kdt = KDTree(registered_geom)
452
+ _, reg_channels = kdt.query(reg_pos)
440
453
  reg_channels = np.atleast_1d(reg_channels)
441
454
 
442
455
  return reg_channels
@@ -59,7 +59,9 @@ def get_clusterer(
59
59
  else:
60
60
  clus_strategy = "none"
61
61
 
62
- saving_labels = save_cfg is not None and save_cfg.save_intermediate_labels
62
+ saving_labels = save_labels_dir is not None and (
63
+ save_cfg is None or save_cfg.save_intermediate_labels
64
+ )
63
65
  if not saving_labels:
64
66
  save_labels_dir = save_cfg = None
65
67
 
@@ -243,7 +243,7 @@ class StableWaveformFeatures:
243
243
  # n, nc_extract
244
244
  channels: Tensor
245
245
  # n, rank, nc_extract
246
- features: Tensor
246
+ features: Tensor | np.ndarray
247
247
  neighborhoods: SpikeNeighborhoods
248
248
  erp: StableFeaturesInterpolator
249
249
 
@@ -292,6 +292,9 @@ class StableWaveformFeatures:
292
292
  trim_to_rank=clustering_features_cfg.feature_rank,
293
293
  params=clustering_features_cfg.interp_params.normalize(),
294
294
  device=computation_cfg.actual_device(),
295
+ store_on_disk=computation_cfg.low_memory,
296
+ tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
297
+ nan_to_num=True,
295
298
  )
296
299
  else:
297
300
  assert sorting.parent_h5_path is not None
@@ -308,13 +311,18 @@ class StableWaveformFeatures:
308
311
  trim_to_rank=clustering_features_cfg.feature_rank,
309
312
  params=clustering_features_cfg.interp_params.normalize(),
310
313
  device=computation_cfg.actual_device(),
314
+ store_on_disk=computation_cfg.low_memory,
315
+ tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
316
+ nan_to_num=True,
311
317
  )
312
318
 
313
- logger.dartsortdebug(f"StableWaveformFeatures {features.shape=}.")
319
+ logger.dartsortdebug(
320
+ f"StableWaveformFeatures {features.shape=} {computation_cfg.low_memory=}."
321
+ )
314
322
 
315
323
  return cls(
316
324
  channels=channels,
317
- features=torch.asarray(features),
325
+ features=features,
318
326
  neighborhoods=spike_neighborhoods,
319
327
  erp=erp,
320
328
  )
@@ -323,6 +331,17 @@ class StableWaveformFeatures:
323
331
  # -- helpers
324
332
 
325
333
 
334
+ def _read_ix_maybe_mmap(x: Tensor | np.ndarray, ix: Tensor | np.ndarray | slice) -> Tensor:
335
+ if isinstance(x, Tensor):
336
+ return x[ix]
337
+ if isinstance(ix, slice):
338
+ return torch.from_numpy(np.array(x[ix]))
339
+ if isinstance(ix, Tensor):
340
+ ix = ix.numpy(force=True)
341
+ assert ix.ndim == 1
342
+ return torch.from_numpy(x[ix])
343
+
344
+
326
345
  def _interpolate_main_channel_features(
327
346
  *,
328
347
  sorting: DARTsortSorting,
@@ -15,6 +15,7 @@ except ImportError:
15
15
  HAVE_CUPY = False
16
16
 
17
17
 
18
+ from ..util.internal_config import SplitKmeansppMetric
18
19
  from ..util.logging_util import DARTSORTDEBUG, get_logger, progrange
19
20
  from ..util.sparse_util import (
20
21
  coo_to_cupy,
@@ -23,7 +24,11 @@ from ..util.sparse_util import (
23
24
  logsumexp_coo,
24
25
  sparse_centroid_distsq,
25
26
  )
26
- from ..util.spiketorch import spawn_torch_rg, sqeuc_cdist_known_norm
27
+ from ..util.spiketorch import (
28
+ scaled_normeuc_cdist_from_dots_,
29
+ spawn_torch_rg,
30
+ sqeuc_cdist_known_norm,
31
+ )
27
32
 
28
33
  logger = get_logger(__name__)
29
34
 
@@ -631,6 +636,8 @@ def batched_kmeans(
631
636
  with_labels=True,
632
637
  with_proportions=True,
633
638
  beta: float = 1.0,
639
+ kmeanspp_metric: SplitKmeansppMetric = "euc",
640
+ scale_dist_args: tuple[float, float, float] | None = None,
634
641
  ) -> KMeansResult:
635
642
  """
636
643
  Compared to above:
@@ -639,6 +646,7 @@ def batched_kmeans(
639
646
  - no weights allowed for now
640
647
  - kmeanspp always random initial
641
648
  - n_iter > 0
649
+ - kmeanspp_metric for seeding only
642
650
  """
643
651
  k = n_components
644
652
  del n_components
@@ -660,9 +668,17 @@ def batched_kmeans(
660
668
  centroid_ixs = torch.full((k, n_kmeanspps), n, dtype=torch.long, device=dev)
661
669
  centroid_ixs[0] = torch.randint(n, size=(n_kmeanspps,), device=dev, generator=gen)
662
670
  dists = X.new_empty((n_kmeanspps, n))
671
+ if kmeanspp_metric == "euc":
672
+ seed_bufs = None
673
+ elif kmeanspp_metric == "scaled_normeuc":
674
+ seed_bufs = X.new_empty((4, n_kmeanspps, n))
675
+ else:
676
+ panic(kmeanspp_metric)
663
677
  Y = X[centroid_ixs[0]]
664
678
  Ynormsq = Xnormsq[centroid_ixs[0]]
665
- dists = sqeuc_cdist_known_norm(Y, Ynormsq, X, Xnormsq, dists)
679
+ dists = _kmeanspp_distsq(
680
+ Y, Ynormsq, X, Xnormsq, dists, seed_bufs, kmeanspp_metric, scale_dist_args
681
+ )
666
682
 
667
683
  # -- kmeanspp stage: loop
668
684
  # buf for random sampling with Gumbel trick and new distance storage
@@ -680,8 +696,11 @@ def batched_kmeans(
680
696
  torch.index_select(Xnormsq, dim=0, index=cix_j, out=Ynormsq)
681
697
 
682
698
  # update distances
683
- newdists = sqeuc_cdist_known_norm(Y, Ynormsq, X, Xnormsq, _buf)
699
+ newdists = _kmeanspp_distsq(
700
+ Y, Ynormsq, X, Xnormsq, _buf, seed_bufs, kmeanspp_metric, scale_dist_args
701
+ )
684
702
  torch.minimum(dists, newdists, out=dists)
703
+ del seed_bufs
685
704
 
686
705
  # -- kmeanspp finish: pick best by phi
687
706
  if kmeanspp_seeds_per_try > 1:
@@ -754,6 +773,40 @@ def batched_kmeans(
754
773
  return KMeansResult(centroids=Y, responsibilities=e, labels=labels, dists=dists)
755
774
 
756
775
 
776
+ def _kmeanspp_distsq(
777
+ Y: Tensor,
778
+ Ynormsq: Tensor,
779
+ X: Tensor,
780
+ Xnormsq: Tensor,
781
+ out: Tensor,
782
+ bufs: Tensor | None,
783
+ metric: SplitKmeansppMetric,
784
+ scale_dist_args: tuple[float, float, float] | None,
785
+ ) -> Tensor:
786
+ if metric == "euc":
787
+ assert bufs is None
788
+ return sqeuc_cdist_known_norm(Y, Ynormsq, X, Xnormsq, out)
789
+ elif metric == "scaled_normeuc":
790
+ assert bufs is not None
791
+ assert scale_dist_args is not None
792
+ scale_std, scale_min, scale_max = scale_dist_args
793
+ dots = torch.mm(Y, X.T, out=bufs[0])
794
+ out = scaled_normeuc_cdist_from_dots_(
795
+ dots,
796
+ Ynormsq,
797
+ Xnormsq,
798
+ out=out,
799
+ bufs=bufs[1:],
800
+ scale_var=scale_std**2,
801
+ scale_min=scale_min,
802
+ scale_max=scale_max,
803
+ )
804
+ assert out.isfinite().all()
805
+ return out.square_()
806
+ else:
807
+ panic(metric)
808
+
809
+
757
810
  def truncated_kmeans_from_labels(
758
811
  X: Tensor | np.ndarray,
759
812
  labels: Tensor | np.ndarray,