DARTsort 0.5.32__tar.gz → 0.5.33__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. {dartsort-0.5.32 → dartsort-0.5.33}/.github/workflows/docs.yml +7 -5
  2. {dartsort-0.5.32 → dartsort-0.5.33}/PKG-INFO +1 -1
  3. {dartsort-0.5.32 → dartsort-0.5.33}/environment.yml +1 -1
  4. {dartsort-0.5.32 → dartsort-0.5.33}/notebook/fit_and_batch_matching.py +100 -32
  5. {dartsort-0.5.32 → dartsort-0.5.33}/notebook/ibl_dandi_example.py +16 -18
  6. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/__init__.py +2 -0
  7. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/clustering_features.py +22 -3
  8. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/mixture.py +52 -34
  9. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/refine_util.py +3 -3
  10. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/config.py +3 -0
  11. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/main.py +12 -6
  12. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/reduction.py +23 -22
  13. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/data_util.py +80 -16
  14. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/internal_config.py +5 -1
  15. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/interpolation_util.py +22 -5
  16. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/motion.py +41 -31
  17. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/py_util.py +11 -0
  18. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/PKG-INFO +1 -1
  19. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_modeling.py +25 -0
  20. {dartsort-0.5.32 → dartsort-0.5.33}/.github/workflows/ci.yml +0 -0
  21. {dartsort-0.5.32 → dartsort-0.5.33}/.github/workflows/ci_windows.yml +0 -0
  22. {dartsort-0.5.32 → dartsort-0.5.33}/.github/workflows/deploy.yml +0 -0
  23. {dartsort-0.5.32 → dartsort-0.5.33}/.gitignore +0 -0
  24. {dartsort-0.5.32 → dartsort-0.5.33}/README.md +0 -0
  25. {dartsort-0.5.32 → dartsort-0.5.33}/docs/index.md +0 -0
  26. {dartsort-0.5.32 → dartsort-0.5.33}/docs/main_api.md +0 -0
  27. {dartsort-0.5.32 → dartsort-0.5.33}/docs/toolkit_api.md +0 -0
  28. {dartsort-0.5.32 → dartsort-0.5.33}/notebook/.keep +0 -0
  29. {dartsort-0.5.32 → dartsort-0.5.33}/pyproject.toml +0 -0
  30. {dartsort-0.5.32 → dartsort-0.5.33}/setup.cfg +0 -0
  31. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/__main__.py +0 -0
  32. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/cli.py +0 -0
  33. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/__init__.py +0 -0
  34. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/agglomerate.py +0 -0
  35. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/cluster_util.py +0 -0
  36. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/clustering.py +0 -0
  37. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/density.py +0 -0
  38. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/forward_backward.py +0 -0
  39. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/kmeans.py +0 -0
  40. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/merge.py +0 -0
  41. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/detect/__init__.py +0 -0
  42. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/detect/detect.py +0 -0
  43. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/__init__.py +0 -0
  44. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/analysis.py +0 -0
  45. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/comparison.py +0 -0
  46. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/hybrid_util.py +0 -0
  47. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/hybridkit.py +0 -0
  48. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  49. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  50. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/simkit.py +0 -0
  51. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/simlib.py +0 -0
  52. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/localize/__init__.py +0 -0
  53. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/localize/localize_torch.py +0 -0
  54. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/localize/localize_util.py +0 -0
  55. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/__init__.py +0 -0
  56. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/grab.py +0 -0
  57. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching.py +0 -0
  58. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/__init__.py +0 -0
  59. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  60. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/drifty.py +0 -0
  61. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  62. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  63. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  64. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/peel_base.py +0 -0
  65. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/peel_lib.py +0 -0
  66. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/reduction_template.py +0 -0
  67. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/shaver.py +0 -0
  68. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/subtract.py +0 -0
  69. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/subtract_util.py +0 -0
  70. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/threshold.py +0 -0
  71. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  72. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/detect_np1.pt +0 -0
  73. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/detect_np2.pt +0 -0
  74. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  75. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  76. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  77. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  78. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  79. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  80. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/__init__.py +0 -0
  81. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/get_templates.py +0 -0
  82. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/postprocess_util.py +0 -0
  83. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/realignment.py +0 -0
  84. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/template_util.py +0 -0
  85. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/templates.py +0 -0
  86. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/templib.py +0 -0
  87. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/__init__.py +0 -0
  88. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  89. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/all_transformers.py +0 -0
  90. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/amortized_localization.py +0 -0
  91. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/amplitudes.py +0 -0
  92. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/decollider.py +0 -0
  93. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/denoising_scorer.py +0 -0
  94. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/enforce_decrease.py +0 -0
  95. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/fixed_prop.py +0 -0
  96. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/interp.py +0 -0
  97. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/localize.py +0 -0
  98. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/matched_filter_net.py +0 -0
  99. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/matching_denoiser.py +0 -0
  100. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/mixture_classifier.py +0 -0
  101. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/pipeline.py +0 -0
  102. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  103. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/supervised_denoiser.py +0 -0
  104. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/temporal_pca.py +0 -0
  105. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/transform_base.py +0 -0
  106. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/vq_matched_filter.py +0 -0
  107. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/whiten.py +0 -0
  108. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/__init__.py +0 -0
  109. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/cli_util.py +0 -0
  110. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/drift_util.py +0 -0
  111. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/job_util.py +0 -0
  112. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/logging_util.py +0 -0
  113. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/main_util.py +0 -0
  114. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/more_operators.py +0 -0
  115. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/multiprocessing_util.py +0 -0
  116. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/nn_util.py +0 -0
  117. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/noise_util.py +0 -0
  118. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/peel_util.py +0 -0
  119. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/preprocess_util.py +0 -0
  120. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/registration_util.py +0 -0
  121. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/residual_util.py +0 -0
  122. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/sparse_util.py +0 -0
  123. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/spikeio.py +0 -0
  124. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/spiketorch.py +0 -0
  125. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/testing_util/__init__.py +0 -0
  126. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  127. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  128. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/torch_optimization_util.py +0 -0
  129. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/torch_util.py +0 -0
  130. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/waveform_util.py +0 -0
  131. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/__init__.py +0 -0
  132. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/analysis_plots.py +0 -0
  133. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/colors.py +0 -0
  134. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/gt.py +0 -0
  135. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/layout.py +0 -0
  136. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/metrics_util.py +0 -0
  137. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/mixture.py +0 -0
  138. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/recanim.py +0 -0
  139. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/scatterplots.py +0 -0
  140. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/sorting.py +0 -0
  141. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/unit.py +0 -0
  142. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/unit_comparison.py +0 -0
  143. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/versus.py +0 -0
  144. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/vismain.py +0 -0
  145. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/waveforms.py +0 -0
  146. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/SOURCES.txt +0 -0
  147. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/dependency_links.txt +0 -0
  148. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/entry_points.txt +0 -0
  149. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/requires.txt +0 -0
  150. {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/top_level.txt +0 -0
  151. {dartsort-0.5.32 → dartsort-0.5.33}/tests/conftest.py +0 -0
  152. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_agglomerate.py +0 -0
  153. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_alignment.py +0 -0
  154. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_cluster_util.py +0 -0
  155. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_clustering.py +0 -0
  156. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_clustering_features.py +0 -0
  157. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_config.py +0 -0
  158. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_dartsort.py +0 -0
  159. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_data_util.py +0 -0
  160. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_density.py +0 -0
  161. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_detect.py +0 -0
  162. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_drift_util.py +0 -0
  163. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_drifty_matching.py +0 -0
  164. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_enforce_decrease.py +0 -0
  165. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_grab_and_featurize.py +0 -0
  166. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_hybridkit.py +0 -0
  167. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_kmeans.py +0 -0
  168. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_linalg.py +0 -0
  169. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_localize_torch.py +0 -0
  170. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_matching.py +0 -0
  171. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_mixture_classifier.py +0 -0
  172. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_multiprocessing_util.py +0 -0
  173. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_simkit.py +0 -0
  174. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_spiketorch.py +0 -0
  175. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_subtract.py +0 -0
  176. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_templates.py +0 -0
  177. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_threshold.py +0 -0
  178. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_torch_optimization_util.py +0 -0
  179. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_transform.py +0 -0
  180. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_util.py +0 -0
  181. {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_waveform_util.py +0 -0
  182. {dartsort-0.5.32 → dartsort-0.5.33}/zensical.toml +0 -0
@@ -29,16 +29,18 @@ jobs:
29
29
  # this is a "fine-grained token"
30
30
  token: ${{ secrets.DARTSORT_GITHUB_IO }}
31
31
 
32
- - name: Setup Python
33
- uses: actions/setup-python@v6
32
+ - name: Setup micromamba
33
+ uses: mamba-org/setup-micromamba@v3
34
34
  with:
35
- python-version: 3.x
36
- cache: "pip" # caching pip dependencies
35
+ environment-file: dartsort/environment.yml
36
+ environment-name: dartsort-docs
37
37
 
38
38
  - name: pip install stuff
39
- run: python3 -m pip install -e dartsort/[doc]
39
+ shell: bash -el {0}
40
+ run: python -m pip install -e "dartsort/[doc]"
40
41
 
41
42
  - name: Zensical build
43
+ shell: bash -el {0}
42
44
  run: pushd dartsort/ && zensical build --clean && popd
43
45
 
44
46
  - name: overwrite site
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.32
3
+ Version: 0.5.33
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -3,7 +3,7 @@ name: dartsort
3
3
  channels:
4
4
  - conda-forge
5
5
  dependencies:
6
- - python>=3.11
6
+ - python>=3.12
7
7
  - h5py
8
8
  - ipykernel
9
9
  - ipython
@@ -5,11 +5,11 @@
5
5
  # extension: .py
6
6
  # format_name: percent
7
7
  # format_version: '1.3'
8
- # jupytext_version: 1.19.5
8
+ # jupytext_version: 1.19.6
9
9
  # kernelspec:
10
- # display_name: Python [conda env:dart14]
10
+ # display_name: Python [conda env:a]
11
11
  # language: python
12
- # name: conda-env-dart14-py
12
+ # name: conda-env-a-py
13
13
  # ---
14
14
 
15
15
  # %% [markdown]
@@ -31,18 +31,16 @@
31
31
  # recording.
32
32
 
33
33
  # %%
34
- import matplotlib.pyplot as plt
35
-
36
- # %%
37
- # dandi-related
38
- from dandi.dandiapi import DandiAPIClient
39
- import fsspec
34
+ import dataclasses
40
35
 
41
- # %%
36
+ import matplotlib.pyplot as plt
42
37
  import numpy as np
43
38
  import spikeinterface.full as si
39
+ from dandi.dandiapi import DandiAPIClient
40
+
44
41
  import dartsort
45
42
  import dartsort.vis as dartvis
43
+
46
44
  dartvis.set_plt_style()
47
45
 
48
46
  # %% [markdown]
@@ -50,12 +48,10 @@ dartvis.set_plt_style()
50
48
 
51
49
  # %%
52
50
  # update this for your machine!
53
- experiment_path = dartsort.ensure_path(
54
- "~/scratch/ibl-dandi-example/", mkdir=True
55
- )
51
+ experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
56
52
 
57
53
  # %%
58
- ds_dir = experiment_path / "dartsort"
54
+ ds_dir = experiment_path / "dartsort-chunked"
59
55
 
60
56
  # %%
61
57
  # IBL BWM dandiset, example recording
@@ -93,7 +89,7 @@ else:
93
89
  print(rec0)
94
90
 
95
91
  # slice 10 minutes near the end
96
- #TODO better IBL sampling rate handling
92
+ # TODO better IBL sampling rate handling
97
93
  rec = rec0
98
94
  end_time = rec.get_end_time()
99
95
  start_frame = rec.time_to_sample_index(end_time - 30 * 60)
@@ -103,36 +99,109 @@ else:
103
99
  rec._sampling_frequency = 30_000
104
100
 
105
101
  rec = dartsort.preprocess(rec, strategy="ibllike")
106
- rec = rec.save_to_folder(preprocessing_path)
102
+ rec = rec.save(folder=preprocessing_path, format="binary")
107
103
 
108
104
  # %%
109
105
  plt.imshow(
110
106
  rec.get_traces(0, 10_000, 11_000).T,
111
- aspect='auto',
107
+ aspect="auto",
112
108
  vmin=-5,
113
109
  vmax=5,
114
- interpolation='nearest',
110
+ interpolation="nearest",
115
111
  )
116
- plt.colorbar(shrink=0.5, label='standardized voltage')
117
- plt.xlabel('time (samples')
118
- plt.ylabel('channels');
119
-
112
+ plt.colorbar(shrink=0.5, label="standardized voltage")
113
+ plt.xlabel("time (samples")
114
+ plt.ylabel("channels")
120
115
  # %% [markdown]
121
116
  # # Configuration
122
117
 
123
118
  # %%
124
119
  # use defaults, except for the pipeline change
125
- cfg = dartsort.DARTsortUserConfig(fit_matching_models_only=True)
120
+ cfg = dartsort.DARTsortUserConfig()
126
121
 
127
122
  # %%
128
123
  # process into an internal config object for later use
129
124
  internal_cfg = dartsort.to_internal_config(cfg)
130
125
 
131
126
  # %% [markdown]
132
- # # Fit models
127
+ # # Initial spike detection and motion tracking model fits
133
128
 
134
129
  # %%
135
- dartsort.dartsort(rec, ds_dir, cfg=cfg)
130
+ dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_motion_models_only=True))
131
+
132
+ # %% [markdown]
133
+ # # Chunked spike detection for motion tracking
134
+ #
135
+ # While dartsort itself only needs to detect spikes on a subsample of the recording
136
+ # in order to do clustering, motion tracking does require spikes from the whole
137
+ # recording. dartsort uses a cheap but good spike detection and localization pass for
138
+ # this, which is very fast, but users at the thousand hour scale still might want to
139
+ # parallelize it.
140
+ #
141
+ # This runs that cheap spike detection for motion tracking in 1-minute batches for demo
142
+ # purposes.
143
+ #
144
+ # Here, the `chunk_starts_samples` argument slices the recording for us.
145
+ # This is better in this case than using `.frame_slice` or `.time_slice` on the recording,
146
+ # since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
147
+ #
148
+ # If the initial detection already covered the whole recording, motion tracking uses its
149
+ # spikes and this step is skipped.
150
+
151
+ # %%
152
+ subtraction_sorting = dartsort.load(ds_dir / "subtraction.h5")
153
+ need_motion_peaks = cfg.do_motion_estimation and dartsort.motion_needs_peaks(
154
+ internal_cfg, rec, subtraction_sorting
155
+ )
156
+ need_motion_peaks
157
+
158
+ # %%
159
+ chunk_length_samples = internal_cfg.motion_estimation_cfg.threshold_cfg.chunk_length_samples
160
+ batch_length_samples = 60 * chunk_length_samples
161
+ duration_samples = rec.get_num_samples()
162
+
163
+ if need_motion_peaks:
164
+ for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, batch_length_samples)):
165
+ batch_end_samples = min(duration_samples, batch_start_samples + batch_length_samples)
166
+ res = dartsort.detect_for_motion(
167
+ recording=rec,
168
+ output_dir=ds_dir,
169
+ previous_sorting=subtraction_sorting,
170
+ hdf5_filename=f"motionthreshold_batch{batch_ix:05}.h5",
171
+ chunk_starts_samples=np.arange(batch_start_samples, batch_end_samples, chunk_length_samples),
172
+ motion_cfg=internal_cfg.motion_estimation_cfg,
173
+ computation_cfg=internal_cfg.motion_computation_cfg,
174
+ sampling_cfg=internal_cfg.peeler_sampling_cfg,
175
+ waveform_cfg=internal_cfg.waveform_cfg,
176
+ )
177
+ del res
178
+
179
+ # %% [markdown]
180
+ # Combine the output of those detections into one sorting, and check that together the
181
+ # batches covered the recording.
182
+
183
+ # %%
184
+ if need_motion_peaks and not (ds_dir / "motionthreshold.h5").exists():
185
+ motion_chunk_h5s = sorted(ds_dir.glob("motionthreshold_batch*.h5"))
186
+ dartsort.virtual_concatenate_sorting_h5s(motion_chunk_h5s, ds_dir / "motionthreshold.h5")
187
+ motion_sorting = dartsort.load(ds_dir / "motionthreshold.h5")
188
+ assert np.array_equal(
189
+ motion_sorting.load_dataset("chunk_starts_samples"),
190
+ np.arange(0, duration_samples, chunk_length_samples),
191
+ )
192
+ print(motion_sorting)
193
+
194
+ # %%
195
+ # !ls -lah {ds_dir}
196
+
197
+ # %% [markdown]
198
+ # ^ Notice all of the chunked motion files! But motionthreshold.h5 itself is tiny, because the concatenation is virtual.
199
+
200
+ # %% [markdown]
201
+ # # Resume: estimate motion, cluster, learn templates and the matching step classifier
202
+
203
+ # %%
204
+ dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_matching_models_only=True))
136
205
 
137
206
  # %%
138
207
  # !ls {ds_dir}
@@ -146,11 +215,8 @@ dartsort.dartsort(rec, ds_dir, cfg=cfg)
146
215
  # %% [markdown]
147
216
  # # Chunked template matching
148
217
  #
149
- # This runs template matching in 1-minute batches.
150
- #
151
- # Here, `chunk_starts_samples` slices the recording for us.
152
- # This might be better than using `.frame_slice` or `.time_slice` on the recording,
153
- # since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
218
+ # This runs template matching in 1-minute batches, with the same loop over chunks
219
+ # as the motion tracking's spike detection above.
154
220
  #
155
221
 
156
222
  # %%
@@ -186,7 +252,9 @@ for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, 60 * 3
186
252
  # which can be turned off for users who don't need them.
187
253
 
188
254
  # %%
189
- sorting = dartsort.concatenate_sortings([dartsort.load(h5) for h5 in sorted(ds_dir.glob("matching1_batch*.h5"))])
255
+ sorting = dartsort.concatenate_sortings(
256
+ [dartsort.load(h5) for h5 in sorted(ds_dir.glob("matching1_batch*.h5"))]
257
+ )
190
258
 
191
259
  # %%
192
260
  sorting = dartsort.cluster(
@@ -199,7 +267,7 @@ sorting = dartsort.cluster(
199
267
 
200
268
  # %%
201
269
  # if you want to keep this...
202
- sorting.save(ds_dir / 'dartsort_sorting.npz')
270
+ sorting.save(ds_dir / "dartsort_sorting.npz")
203
271
 
204
272
  # %% [markdown]
205
273
  # # Visualize
@@ -5,25 +5,26 @@
5
5
  # extension: .py
6
6
  # format_name: percent
7
7
  # format_version: '1.3'
8
- # jupytext_version: 1.19.3
8
+ # jupytext_version: 1.19.6
9
9
  # kernelspec:
10
- # display_name: Python [conda env:dart14]
10
+ # display_name: Python [conda env:a]
11
11
  # language: python
12
- # name: conda-env-dart14-py
12
+ # name: conda-env-a-py
13
13
  # ---
14
14
 
15
15
  # %%
16
16
  import matplotlib.pyplot as plt
17
17
 
18
+ # %%
19
+ import spikeinterface.full as si
20
+
18
21
  # %%
19
22
  # dandi-related
20
23
  from dandi.dandiapi import DandiAPIClient
21
- import fsspec
22
24
 
23
- # %%
24
- import spikeinterface.full as si
25
25
  import dartsort
26
26
  import dartsort.vis as dartvis
27
+
27
28
  dartvis.set_plt_style()
28
29
 
29
30
  # %% [markdown]
@@ -31,9 +32,7 @@ dartvis.set_plt_style()
31
32
 
32
33
  # %%
33
34
  # update this for your machine!
34
- experiment_path = dartsort.ensure_path(
35
- "~/scratch/ibl-dandi-example/", mkdir=True
36
- )
35
+ experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
37
36
 
38
37
  # %%
39
38
  # IBL BWM dandiset, example recording
@@ -71,7 +70,7 @@ else:
71
70
  print(rec0)
72
71
 
73
72
  # slice 10 minutes near the end
74
- #TODO better IBL sampling rate handling
73
+ # TODO better IBL sampling rate handling
75
74
  rec = rec0
76
75
  end_time = rec.get_end_time()
77
76
  start_frame = rec.time_to_sample_index(end_time - 30 * 60)
@@ -81,20 +80,19 @@ else:
81
80
  rec._sampling_frequency = 30_000
82
81
 
83
82
  rec = dartsort.preprocess(rec, strategy="ibllike")
84
- rec = rec.save_to_folder(preprocessing_path)
83
+ rec = rec.save(folder=preprocessing_path, format="binary")
85
84
 
86
85
  # %%
87
86
  plt.imshow(
88
87
  rec.get_traces(0, 10_000, 11_000).T,
89
- aspect='auto',
88
+ aspect="auto",
90
89
  vmin=-5,
91
90
  vmax=5,
92
- interpolation='nearest',
91
+ interpolation="nearest",
93
92
  )
94
- plt.colorbar(shrink=0.5, label='standardized voltage')
95
- plt.xlabel('time (samples')
96
- plt.ylabel('channels');
97
-
93
+ plt.colorbar(shrink=0.5, label="standardized voltage")
94
+ plt.xlabel("time (samples")
95
+ plt.ylabel("channels")
98
96
  # %% [markdown]
99
97
  # ## Run *dartsort*
100
98
 
@@ -106,6 +104,6 @@ ds_res = dartsort.dartsort(rec, ds_dir)
106
104
  # # Visualize
107
105
 
108
106
  # %%
109
- dartvis.scatter_spike_features(sorting=ds_res["sorting"]);
107
+ dartvis.scatter_spike_features(sorting=ds_res["sorting"])
110
108
 
111
109
  # %%
@@ -63,6 +63,7 @@ from .util.data_util import (
63
63
  sorting_from_spikeinterface,
64
64
  try_get_denoising_pipeline,
65
65
  try_get_model_dir,
66
+ virtual_concatenate_sorting_h5s,
66
67
  )
67
68
  from .util.drift_util import registered_geometry
68
69
  from .util.internal_config import *
@@ -80,6 +81,7 @@ from .util.logging_util import (
80
81
  get_logger,
81
82
  set_log_level,
82
83
  )
84
+ from .util.main_util import motion_needs_peaks
83
85
  from .util.motion import (
84
86
  MotionInfo,
85
87
  detect_for_motion,
@@ -243,7 +243,7 @@ class StableWaveformFeatures:
243
243
  # n, nc_extract
244
244
  channels: Tensor
245
245
  # n, rank, nc_extract
246
- features: Tensor
246
+ features: Tensor | np.ndarray
247
247
  neighborhoods: SpikeNeighborhoods
248
248
  erp: StableFeaturesInterpolator
249
249
 
@@ -292,6 +292,9 @@ class StableWaveformFeatures:
292
292
  trim_to_rank=clustering_features_cfg.feature_rank,
293
293
  params=clustering_features_cfg.interp_params.normalize(),
294
294
  device=computation_cfg.actual_device(),
295
+ store_on_disk=computation_cfg.low_memory,
296
+ tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
297
+ nan_to_num=True,
295
298
  )
296
299
  else:
297
300
  assert sorting.parent_h5_path is not None
@@ -308,13 +311,18 @@ class StableWaveformFeatures:
308
311
  trim_to_rank=clustering_features_cfg.feature_rank,
309
312
  params=clustering_features_cfg.interp_params.normalize(),
310
313
  device=computation_cfg.actual_device(),
314
+ store_on_disk=computation_cfg.low_memory,
315
+ tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
316
+ nan_to_num=True,
311
317
  )
312
318
 
313
- logger.dartsortdebug(f"StableWaveformFeatures {features.shape=}.")
319
+ logger.dartsortdebug(
320
+ f"StableWaveformFeatures {features.shape=} {computation_cfg.low_memory=}."
321
+ )
314
322
 
315
323
  return cls(
316
324
  channels=channels,
317
- features=torch.asarray(features),
325
+ features=features,
318
326
  neighborhoods=spike_neighborhoods,
319
327
  erp=erp,
320
328
  )
@@ -323,6 +331,17 @@ class StableWaveformFeatures:
323
331
  # -- helpers
324
332
 
325
333
 
334
+ def _read_ix_maybe_mmap(x: Tensor | np.ndarray, ix: Tensor | np.ndarray | slice) -> Tensor:
335
+ if isinstance(x, Tensor):
336
+ return x[ix]
337
+ if isinstance(ix, slice):
338
+ return torch.from_numpy(np.array(x[ix]))
339
+ if isinstance(ix, Tensor):
340
+ ix = ix.numpy(force=True)
341
+ assert ix.ndim == 1
342
+ return torch.from_numpy(x[ix])
343
+
344
+
326
345
  def _interpolate_main_channel_features(
327
346
  *,
328
347
  sorting: DARTsortSorting,
@@ -109,7 +109,7 @@ from ..util.spiketorch import (
109
109
  )
110
110
  from ..util.torch_util import BModule, cleanup_and_log_gpu_usage, torch_compiler
111
111
  from .cluster_util import decrumb, linkage, maximal_leaf_groups, violation_statistics
112
- from .clustering_features import StableWaveformFeatures
112
+ from .clustering_features import StableWaveformFeatures, _read_ix_maybe_mmap
113
113
  from .kmeans import _one_gumbel_nolog, batched_kmeans, kmeans
114
114
 
115
115
  if TYPE_CHECKING:
@@ -1568,7 +1568,7 @@ class StreamingSpikeData(BatchedSpikeData):
1568
1568
  *,
1569
1569
  n_candidates: int,
1570
1570
  max_n_candidates: int,
1571
- x: Tensor,
1571
+ x: Tensor | np.ndarray,
1572
1572
  duties: Tensor | None,
1573
1573
  neighborhoods: SpikeNeighborhoods,
1574
1574
  device: torch.device,
@@ -1660,7 +1660,7 @@ class StreamingSpikeData(BatchedSpikeData):
1660
1660
  duties = None
1661
1661
  else:
1662
1662
  duties = self.duties[spike_indices].to(self.device)
1663
- x = self.x[spike_indices].to(self.device)
1663
+ x = _read_ix_maybe_mmap(self.x, spike_indices).to(self.device)
1664
1664
  wx, noise_loglik = _whiten_and_noise_score_batch(
1665
1665
  x=x, neighb_ids=neighb_ids, neighb_cov=self.neighb_cov
1666
1666
  )
@@ -1753,7 +1753,12 @@ class TruncatedSpikeData(BatchedSpikeData):
1753
1753
  assert self.storage_device in (device, torch.device("cpu"))
1754
1754
  for f in (xt, whitenedx, CmoCooinvx, noise_logliks):
1755
1755
  assert f is None or f.device == self.storage_device
1756
- assert self.storage_device == device or f is None or f.is_pinned()
1756
+ assert (
1757
+ self.storage_device == device
1758
+ or f is None
1759
+ or not f.numel()
1760
+ or f.is_pinned()
1761
+ )
1757
1762
 
1758
1763
  self.score_only = xt is None
1759
1764
  assert self.score_only == (CmoCooinvx is None)
@@ -2846,7 +2851,8 @@ class TruncatedMixtureModel(BaseMixtureModel):
2846
2851
  denom = stats.N.clamp_(min=self.eps)
2847
2852
  tikh = self.p.prior_pseudocount / denom
2848
2853
  U.diagonal(dim1=-2, dim2=-1).add_(tikh[:, None])
2849
- soln = torch.linalg.solve(U, stats.R)
2854
+ Uchol, _ = torch.linalg.cholesky_ex(U)
2855
+ soln = torch.cholesky_solve(stats.R, Uchol)
2850
2856
  self.b.means.copy_(soln[:, -1])
2851
2857
  self.b.bases.copy_(soln[:, :-1])
2852
2858
 
@@ -4544,6 +4550,7 @@ def get_truncated_datasets(
4544
4550
  noise: EmbeddedNoise | None = None,
4545
4551
  stable_features: StableWaveformFeatures | None = None,
4546
4552
  save_kwargs: dict | None = None,
4553
+ computation_cfg: ComputationConfig | None = None,
4547
4554
  ):
4548
4555
  # we assume that the core neighborhoods are exactly the same as extract ones
4549
4556
  data = get_full_neighborhood_data(
@@ -4555,6 +4562,7 @@ def get_truncated_datasets(
4555
4562
  fit_indices=fit_indices,
4556
4563
  stable_features=stable_features,
4557
4564
  device=device,
4565
+ computation_cfg=computation_cfg,
4558
4566
  )
4559
4567
  feature_rank, full_features, full_neighbs, train_ixs, train_neighbs, val_ixs, val_neighbs, prgeom, train_weights, val_weights = data # fmt: skip
4560
4568
  del stable_features
@@ -4601,11 +4609,11 @@ def get_truncated_datasets(
4601
4609
  huber_quantile=noise_huber_quantile(refinement_cfg),
4602
4610
  )
4603
4611
 
4612
+ train_x = _read_ix_maybe_mmap(full_features, train_ixs)
4604
4613
  if refinement_cfg.spike_storage == "device" or not sorting.n_units:
4605
- train_x = full_features[train_ixs].to(device=device)
4614
+ train_x = train_x.to(device=device)
4606
4615
  elif refinement_cfg.spike_storage == "cpu":
4607
- assert full_features.device.type == "cpu"
4608
- train_x = full_features[train_ixs]
4616
+ assert train_x.device.type == "cpu"
4609
4617
  else:
4610
4618
  panic(refinement_cfg.spike_storage)
4611
4619
 
@@ -4712,11 +4720,11 @@ def get_truncated_datasets(
4712
4720
  val_duties = None
4713
4721
  else:
4714
4722
  val_duties = duties[val_ixs].to(device=device)
4723
+ val_x = _read_ix_maybe_mmap(full_features, val_ixs)
4715
4724
  if refinement_cfg.spike_storage == "device":
4716
- val_x = full_features[val_ixs].to(device=device)
4725
+ val_x = val_x.to(device=device)
4717
4726
  elif refinement_cfg.spike_storage == "cpu":
4718
- assert full_features.device.type == "cpu"
4719
- val_x = full_features[val_ixs]
4727
+ assert val_x.device.type == "cpu"
4720
4728
  else:
4721
4729
  panic(refinement_cfg.spike_storage)
4722
4730
  val_data = TruncatedSpikeData.initialize_from_labels(
@@ -4822,8 +4830,7 @@ def get_full_neighborhood_data(
4822
4830
  if clustering_features_cfg is not None:
4823
4831
  assert 0 < frank <= clustering_features_cfg.feature_rank
4824
4832
  assert xfull.shape[0] == full_neighborhoods.b.neighborhood_ids.shape[0]
4825
- xfull = xfull.view(len(xfull), -1)
4826
- xfull = xfull.nan_to_num_()
4833
+ xfull = xfull.reshape(len(xfull), -1)
4827
4834
 
4828
4835
  full_neighborhoods = full_neighborhoods.to(device=device)
4829
4836
 
@@ -5008,6 +5015,7 @@ def instantiate_and_bootstrap_tmm(
5008
5015
  device=device,
5009
5016
  rg=rg,
5010
5017
  save_kwargs=save_kwargs,
5018
+ computation_cfg=computation_cfg,
5011
5019
  )
5012
5020
  )
5013
5021
 
@@ -5153,7 +5161,7 @@ class TruncatedKmeanspp:
5153
5161
  def fit_predict(
5154
5162
  cls,
5155
5163
  *,
5156
- x: Tensor,
5164
+ x: Tensor | np.ndarray,
5157
5165
  train_x: Tensor,
5158
5166
  neighborhoods: SpikeNeighborhoods,
5159
5167
  neighb_cov: NeighborhoodCovariance,
@@ -5235,42 +5243,52 @@ class TruncatedKmeanspp:
5235
5243
  def label(
5236
5244
  self,
5237
5245
  *,
5238
- x: Tensor,
5246
+ x: Tensor | np.ndarray,
5239
5247
  neighborhoods: SpikeNeighborhoods,
5240
5248
  neighb_cov: NeighborhoodCovariance,
5241
5249
  centroid_ixs: Tensor,
5242
5250
  batch_size: int,
5251
+ chunk_size: int = 2**20,
5243
5252
  ) -> Tensor:
5244
5253
  """In batches, label each x by its best centroid"""
5245
5254
  max_nc_obs = self.neighborhoods.shape[1]
5246
5255
  device = self.X.device
5256
+ n_neighbs = neighborhoods.n_neighborhoods
5247
5257
  neighb_ids = neighborhoods.b.neighborhood_ids
5248
5258
  cent_visible = self.visible[:, self.Xneighbixs[centroid_ixs]]
5249
5259
 
5250
- full_labels = torch.full(x.shape[:1], -1, dtype=torch.long, device=device)
5251
- order, indptr, _ = sort_to_csr(neighb_ids, neighborhoods.n_neighborhoods)
5252
- indptr_list = indptr.tolist()
5253
- for ni in range(neighborhoods.n_neighborhoods):
5254
- members = order[indptr_list[ni] : indptr_list[ni + 1]]
5255
- if not members.numel():
5256
- continue
5257
-
5260
+ neighb_centers: dict[int, tuple[Tensor, NeighborhoodCenters]] = {}
5261
+ for ni in range(n_neighbs):
5258
5262
  (cand,) = cent_visible[ni].nonzero(as_tuple=True)
5259
5263
  if not cand.numel():
5260
5264
  # kmeans++ ran out of components before covering this neighborhood
5261
5265
  continue
5262
- centers = self.centers_in_neighborhood(ni, centroid_ixs[cand])
5266
+ neighb_centers[ni] = cand, self.centers_in_neighborhood(
5267
+ ni, centroid_ixs[cand]
5268
+ )
5263
5269
 
5264
- for bs in range(0, members.numel(), batch_size):
5265
- bix = members[bs : bs + batch_size]
5266
- xb = x[bix.to(x.device)].to(device)
5267
- wx, _ = _whiten_and_noise_score_batch(
5268
- x=xb, neighb_ids=neighb_ids[bix], neighb_cov=neighb_cov
5269
- )
5270
- xsq = wx.view(bix.shape[0], self.feat_rank, max_nc_obs)
5271
- xsq = xsq.square().sum(dim=1)
5272
- d = self.center_dists(wx, xsq, centers)
5273
- full_labels[bix] = cand[d.argmin(dim=1)]
5270
+ full_labels = torch.full(x.shape[:1], -1, dtype=torch.long, device=device)
5271
+ for i0 in range(0, x.shape[0], chunk_size):
5272
+ i1 = min(x.shape[0], i0 + chunk_size)
5273
+ x_chunk = _read_ix_maybe_mmap(x, slice(i0, i1))
5274
+ chunk_neighb_ids = neighb_ids[i0:i1]
5275
+ order, indptr, _ = sort_to_csr(chunk_neighb_ids, n_neighbs)
5276
+ indptr_list = indptr.tolist()
5277
+ for ni, (cand, centers) in neighb_centers.items():
5278
+ members = order[indptr_list[ni] : indptr_list[ni + 1]]
5279
+ for bs in range(0, members.numel(), batch_size):
5280
+ bix = members[bs : bs + batch_size]
5281
+ xb = x_chunk[bix.to(x_chunk.device)].to(device)
5282
+ wx, _ = _whiten_and_noise_score_batch(
5283
+ x=xb,
5284
+ neighb_ids=chunk_neighb_ids[bix],
5285
+ neighb_cov=neighb_cov,
5286
+ )
5287
+ xsq = wx.view(bix.shape[0], self.feat_rank, max_nc_obs)
5288
+ xsq = xsq.square().sum(dim=1)
5289
+ d = self.center_dists(wx, xsq, centers)
5290
+ full_labels[i0 + bix] = cand[d.argmin(dim=1)]
5291
+ del x_chunk
5274
5292
 
5275
5293
  n_unlabeled = (full_labels < 0).sum().item()
5276
5294
  if n_unlabeled:
@@ -18,7 +18,7 @@ from ..util.motion import MotionInfo
18
18
  from ..util.multiprocessing_util import pool_from_cfg
19
19
  from ..util.py_util import databag
20
20
  from .cluster_util import hierarchical_cluster, reorder_by_depth
21
- from .clustering_features import StableWaveformFeatures
21
+ from .clustering_features import StableWaveformFeatures, _read_ix_maybe_mmap
22
22
 
23
23
  logger = get_logger(__name__)
24
24
  HAVE_KDEPY = importlib.util.find_spec("KDEpy") is not None
@@ -63,7 +63,7 @@ def pc_merge(
63
63
 
64
64
  # make stable features, no need for core features though.
65
65
  kept = np.flatnonzero(subset_sorting.labels >= 0)
66
- x = stable_features.features[kept]
66
+ x = _read_ix_maybe_mmap(stable_features.features, kept)
67
67
  x = x[:, : refinement_cfg.pc_merge_rank]
68
68
  xlabels = torch.from_numpy(subset_sorting.labels[kept]).to(x.device)
69
69
  n_reg_chans = motion.rgeom.shape[0]
@@ -456,7 +456,7 @@ def collision_cleaning_errors(
456
456
 
457
457
  # average feature
458
458
  kept = np.flatnonzero(subset_sorting.labels >= 0)
459
- x = stable_features.features[kept]
459
+ x = _read_ix_maybe_mmap(stable_features.features, kept)
460
460
  xlabels = torch.from_numpy(subset_sorting.labels[kept]).to(x.device)
461
461
  n_reg_chans = motion.rgeom.shape[0]
462
462
  means, counts = spiketorch.average_by_label(
@@ -137,6 +137,8 @@ class DARTsortUserConfig:
137
137
  workdir_follow_symlinks: bool = False
138
138
  tmpdir_parent: str | None = None
139
139
  """Control where tmpdirs are created."""
140
+ low_memory: bool = False
141
+ """Best effort at not using tons of RAM. If you notice memory problems, please get in touch!"""
140
142
  save_intermediates: bool = False
141
143
  """Store all spike features from intermediate steps (for debugging)"""
142
144
 
@@ -269,6 +271,7 @@ class DARTsortUserConfig:
269
271
  """Chunk length in seconds for online motion tracking"""
270
272
 
271
273
  # -- pipeline control parameters for specific use cases
274
+ fit_motion_models_only: bool = False
272
275
  fit_matching_models_only: bool = False
273
276
  save_full_final_residual: bool = False
274
277