DARTsort 0.5.32__tar.gz → 0.5.34__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. {dartsort-0.5.32 → dartsort-0.5.34}/.github/workflows/docs.yml +7 -5
  2. {dartsort-0.5.32 → dartsort-0.5.34}/PKG-INFO +1 -2
  3. {dartsort-0.5.32 → dartsort-0.5.34}/environment.yml +1 -2
  4. {dartsort-0.5.32 → dartsort-0.5.34}/notebook/fit_and_batch_matching.py +100 -32
  5. {dartsort-0.5.32 → dartsort-0.5.34}/notebook/ibl_dandi_example.py +16 -18
  6. {dartsort-0.5.32 → dartsort-0.5.34}/pyproject.toml +0 -1
  7. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/__init__.py +3 -1
  8. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/clustering_features.py +22 -3
  9. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/mixture.py +358 -514
  10. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/refine_util.py +3 -14
  11. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/config.py +6 -5
  12. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/comparison.py +1 -1
  13. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/hybridkit.py +9 -6
  14. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/simkit.py +0 -3
  15. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/simlib.py +274 -256
  16. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/main.py +12 -6
  17. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/realignment.py +1 -1
  18. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/mixture_classifier.py +62 -111
  19. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/reduction.py +36 -24
  20. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/data_util.py +80 -16
  21. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/drift_util.py +61 -13
  22. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/internal_config.py +8 -12
  23. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/interpolation_util.py +84 -49
  24. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/motion.py +42 -40
  25. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/multiprocessing_util.py +9 -0
  26. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/noise_util.py +127 -675
  27. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/py_util.py +26 -1
  28. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/spiketorch.py +5 -58
  29. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/testing_util/mixture_testing_util.py +13 -22
  30. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/mixture.py +7 -18
  31. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/unit_comparison.py +6 -6
  32. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/vismain.py +2 -0
  33. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/PKG-INFO +1 -2
  34. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/SOURCES.txt +0 -2
  35. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/requires.txt +0 -1
  36. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_drift_util.py +6 -4
  37. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_modeling.py +32 -11
  38. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_simkit.py +2 -1
  39. dartsort-0.5.32/src/dartsort/util/more_operators.py +0 -274
  40. dartsort-0.5.32/tests/test_linalg.py +0 -83
  41. {dartsort-0.5.32 → dartsort-0.5.34}/.github/workflows/ci.yml +0 -0
  42. {dartsort-0.5.32 → dartsort-0.5.34}/.github/workflows/ci_windows.yml +0 -0
  43. {dartsort-0.5.32 → dartsort-0.5.34}/.github/workflows/deploy.yml +0 -0
  44. {dartsort-0.5.32 → dartsort-0.5.34}/.gitignore +0 -0
  45. {dartsort-0.5.32 → dartsort-0.5.34}/README.md +0 -0
  46. {dartsort-0.5.32 → dartsort-0.5.34}/docs/index.md +0 -0
  47. {dartsort-0.5.32 → dartsort-0.5.34}/docs/main_api.md +0 -0
  48. {dartsort-0.5.32 → dartsort-0.5.34}/docs/toolkit_api.md +0 -0
  49. {dartsort-0.5.32 → dartsort-0.5.34}/notebook/.keep +0 -0
  50. {dartsort-0.5.32 → dartsort-0.5.34}/setup.cfg +0 -0
  51. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/__main__.py +0 -0
  52. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/cli.py +0 -0
  53. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/__init__.py +0 -0
  54. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/agglomerate.py +0 -0
  55. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/cluster_util.py +0 -0
  56. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/clustering.py +0 -0
  57. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/density.py +0 -0
  58. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/forward_backward.py +0 -0
  59. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/kmeans.py +0 -0
  60. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/merge.py +0 -0
  61. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/detect/__init__.py +0 -0
  62. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/detect/detect.py +0 -0
  63. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/__init__.py +0 -0
  64. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/analysis.py +0 -0
  65. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/hybrid_util.py +0 -0
  66. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  67. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  68. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/localize/__init__.py +0 -0
  69. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/localize/localize_torch.py +0 -0
  70. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/localize/localize_util.py +0 -0
  71. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/__init__.py +0 -0
  72. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/grab.py +0 -0
  73. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching.py +0 -0
  74. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/__init__.py +0 -0
  75. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  76. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/drifty.py +0 -0
  77. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  78. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  79. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  80. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/peel_base.py +0 -0
  81. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/peel_lib.py +0 -0
  82. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/reduction_template.py +0 -0
  83. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/shaver.py +0 -0
  84. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/subtract.py +0 -0
  85. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/subtract_util.py +0 -0
  86. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/threshold.py +0 -0
  87. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  88. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/detect_np1.pt +0 -0
  89. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/detect_np2.pt +0 -0
  90. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  91. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  92. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  93. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  94. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  95. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  96. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/__init__.py +0 -0
  97. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/get_templates.py +0 -0
  98. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/postprocess_util.py +0 -0
  99. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/template_util.py +0 -0
  100. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/templates.py +0 -0
  101. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/templib.py +0 -0
  102. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/__init__.py +0 -0
  103. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  104. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/all_transformers.py +0 -0
  105. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/amortized_localization.py +0 -0
  106. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/amplitudes.py +0 -0
  107. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/decollider.py +0 -0
  108. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/denoising_scorer.py +0 -0
  109. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/enforce_decrease.py +0 -0
  110. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/fixed_prop.py +0 -0
  111. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/interp.py +0 -0
  112. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/localize.py +0 -0
  113. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/matched_filter_net.py +0 -0
  114. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/matching_denoiser.py +0 -0
  115. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/pipeline.py +0 -0
  116. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  117. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/supervised_denoiser.py +0 -0
  118. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/temporal_pca.py +0 -0
  119. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/transform_base.py +0 -0
  120. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/vq_matched_filter.py +0 -0
  121. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/whiten.py +0 -0
  122. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/__init__.py +0 -0
  123. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/cli_util.py +0 -0
  124. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/job_util.py +0 -0
  125. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/logging_util.py +0 -0
  126. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/main_util.py +0 -0
  127. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/nn_util.py +0 -0
  128. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/peel_util.py +0 -0
  129. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/preprocess_util.py +0 -0
  130. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/registration_util.py +0 -0
  131. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/residual_util.py +0 -0
  132. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/sparse_util.py +0 -0
  133. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/spikeio.py +0 -0
  134. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/testing_util/__init__.py +0 -0
  135. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  136. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/torch_optimization_util.py +0 -0
  137. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/torch_util.py +0 -0
  138. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/waveform_util.py +0 -0
  139. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/__init__.py +0 -0
  140. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/analysis_plots.py +0 -0
  141. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/colors.py +0 -0
  142. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/gt.py +0 -0
  143. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/layout.py +0 -0
  144. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/metrics_util.py +0 -0
  145. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/recanim.py +0 -0
  146. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/scatterplots.py +0 -0
  147. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/sorting.py +0 -0
  148. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/unit.py +0 -0
  149. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/versus.py +0 -0
  150. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/waveforms.py +0 -0
  151. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/dependency_links.txt +0 -0
  152. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/entry_points.txt +0 -0
  153. {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/top_level.txt +0 -0
  154. {dartsort-0.5.32 → dartsort-0.5.34}/tests/conftest.py +0 -0
  155. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_agglomerate.py +0 -0
  156. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_alignment.py +0 -0
  157. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_cluster_util.py +0 -0
  158. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_clustering.py +0 -0
  159. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_clustering_features.py +0 -0
  160. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_config.py +0 -0
  161. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_dartsort.py +0 -0
  162. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_data_util.py +0 -0
  163. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_density.py +0 -0
  164. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_detect.py +0 -0
  165. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_drifty_matching.py +0 -0
  166. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_enforce_decrease.py +0 -0
  167. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_grab_and_featurize.py +0 -0
  168. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_hybridkit.py +0 -0
  169. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_kmeans.py +0 -0
  170. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_localize_torch.py +0 -0
  171. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_matching.py +0 -0
  172. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_mixture_classifier.py +0 -0
  173. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_multiprocessing_util.py +0 -0
  174. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_spiketorch.py +0 -0
  175. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_subtract.py +0 -0
  176. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_templates.py +0 -0
  177. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_threshold.py +0 -0
  178. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_torch_optimization_util.py +0 -0
  179. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_transform.py +0 -0
  180. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_util.py +0 -0
  181. {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_waveform_util.py +0 -0
  182. {dartsort-0.5.32 → dartsort-0.5.34}/zensical.toml +0 -0
@@ -29,16 +29,18 @@ jobs:
29
29
  # this is a "fine-grained token"
30
30
  token: ${{ secrets.DARTSORT_GITHUB_IO }}
31
31
 
32
- - name: Setup Python
33
- uses: actions/setup-python@v6
32
+ - name: Setup micromamba
33
+ uses: mamba-org/setup-micromamba@v3
34
34
  with:
35
- python-version: 3.x
36
- cache: "pip" # caching pip dependencies
35
+ environment-file: dartsort/environment.yml
36
+ environment-name: dartsort-docs
37
37
 
38
38
  - name: pip install stuff
39
- run: python3 -m pip install -e dartsort/[doc]
39
+ shell: bash -el {0}
40
+ run: python -m pip install -e "dartsort/[doc]"
40
41
 
41
42
  - name: Zensical build
43
+ shell: bash -el {0}
42
44
  run: pushd dartsort/ && zensical build --clean && popd
43
45
 
44
46
  - name: overwrite site
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.32
3
+ Version: 0.5.34
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -13,7 +13,6 @@ Description-Content-Type: text/markdown
13
13
  Requires-Dist: dredge-ephys>=0.4.0
14
14
  Requires-Dist: h5py
15
15
  Requires-Dist: jaxtyping
16
- Requires-Dist: linear_operator
17
16
  Requires-Dist: numba
18
17
  Requires-Dist: numpy>=1.20; python_version < "3.13"
19
18
  Requires-Dist: numpy>=2.0.0; python_version >= "3.13"
@@ -3,12 +3,11 @@ name: dartsort
3
3
  channels:
4
4
  - conda-forge
5
5
  dependencies:
6
- - python>=3.11
6
+ - python>=3.12
7
7
  - h5py
8
8
  - ipykernel
9
9
  - ipython
10
10
  - ipywidgets
11
- - linear_operator
12
11
  - matplotlib
13
12
  - numba
14
13
  - numpy>=2.0
@@ -5,11 +5,11 @@
5
5
  # extension: .py
6
6
  # format_name: percent
7
7
  # format_version: '1.3'
8
- # jupytext_version: 1.19.5
8
+ # jupytext_version: 1.19.6
9
9
  # kernelspec:
10
- # display_name: Python [conda env:dart14]
10
+ # display_name: Python [conda env:a]
11
11
  # language: python
12
- # name: conda-env-dart14-py
12
+ # name: conda-env-a-py
13
13
  # ---
14
14
 
15
15
  # %% [markdown]
@@ -31,18 +31,16 @@
31
31
  # recording.
32
32
 
33
33
  # %%
34
- import matplotlib.pyplot as plt
35
-
36
- # %%
37
- # dandi-related
38
- from dandi.dandiapi import DandiAPIClient
39
- import fsspec
34
+ import dataclasses
40
35
 
41
- # %%
36
+ import matplotlib.pyplot as plt
42
37
  import numpy as np
43
38
  import spikeinterface.full as si
39
+ from dandi.dandiapi import DandiAPIClient
40
+
44
41
  import dartsort
45
42
  import dartsort.vis as dartvis
43
+
46
44
  dartvis.set_plt_style()
47
45
 
48
46
  # %% [markdown]
@@ -50,12 +48,10 @@ dartvis.set_plt_style()
50
48
 
51
49
  # %%
52
50
  # update this for your machine!
53
- experiment_path = dartsort.ensure_path(
54
- "~/scratch/ibl-dandi-example/", mkdir=True
55
- )
51
+ experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
56
52
 
57
53
  # %%
58
- ds_dir = experiment_path / "dartsort"
54
+ ds_dir = experiment_path / "dartsort-chunked"
59
55
 
60
56
  # %%
61
57
  # IBL BWM dandiset, example recording
@@ -93,7 +89,7 @@ else:
93
89
  print(rec0)
94
90
 
95
91
  # slice 10 minutes near the end
96
- #TODO better IBL sampling rate handling
92
+ # TODO better IBL sampling rate handling
97
93
  rec = rec0
98
94
  end_time = rec.get_end_time()
99
95
  start_frame = rec.time_to_sample_index(end_time - 30 * 60)
@@ -103,36 +99,109 @@ else:
103
99
  rec._sampling_frequency = 30_000
104
100
 
105
101
  rec = dartsort.preprocess(rec, strategy="ibllike")
106
- rec = rec.save_to_folder(preprocessing_path)
102
+ rec = rec.save(folder=preprocessing_path, format="binary")
107
103
 
108
104
  # %%
109
105
  plt.imshow(
110
106
  rec.get_traces(0, 10_000, 11_000).T,
111
- aspect='auto',
107
+ aspect="auto",
112
108
  vmin=-5,
113
109
  vmax=5,
114
- interpolation='nearest',
110
+ interpolation="nearest",
115
111
  )
116
- plt.colorbar(shrink=0.5, label='standardized voltage')
117
- plt.xlabel('time (samples')
118
- plt.ylabel('channels');
119
-
112
+ plt.colorbar(shrink=0.5, label="standardized voltage")
113
+ plt.xlabel("time (samples")
114
+ plt.ylabel("channels")
120
115
  # %% [markdown]
121
116
  # # Configuration
122
117
 
123
118
  # %%
124
119
  # use defaults, except for the pipeline change
125
- cfg = dartsort.DARTsortUserConfig(fit_matching_models_only=True)
120
+ cfg = dartsort.DARTsortUserConfig()
126
121
 
127
122
  # %%
128
123
  # process into an internal config object for later use
129
124
  internal_cfg = dartsort.to_internal_config(cfg)
130
125
 
131
126
  # %% [markdown]
132
- # # Fit models
127
+ # # Initial spike detection and motion tracking model fits
133
128
 
134
129
  # %%
135
- dartsort.dartsort(rec, ds_dir, cfg=cfg)
130
+ dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_motion_models_only=True))
131
+
132
+ # %% [markdown]
133
+ # # Chunked spike detection for motion tracking
134
+ #
135
+ # While dartsort itself only needs to detect spikes on a subsample of the recording
136
+ # in order to do clustering, motion tracking does require spikes from the whole
137
+ # recording. dartsort uses a cheap but good spike detection and localization pass for
138
+ # this, which is very fast, but users at the thousand hour scale still might want to
139
+ # parallelize it.
140
+ #
141
+ # This runs that cheap spike detection for motion tracking in 1-minute batches for demo
142
+ # purposes.
143
+ #
144
+ # Here, the `chunk_starts_samples` argument slices the recording for us.
145
+ # This is better in this case than using `.frame_slice` or `.time_slice` on the recording,
146
+ # since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
147
+ #
148
+ # If the initial detection already covered the whole recording, motion tracking uses its
149
+ # spikes and this step is skipped.
150
+
151
+ # %%
152
+ subtraction_sorting = dartsort.load(ds_dir / "subtraction.h5")
153
+ need_motion_peaks = cfg.do_motion_estimation and dartsort.motion_needs_peaks(
154
+ internal_cfg, rec, subtraction_sorting
155
+ )
156
+ need_motion_peaks
157
+
158
+ # %%
159
+ chunk_length_samples = internal_cfg.motion_estimation_cfg.threshold_cfg.chunk_length_samples
160
+ batch_length_samples = 60 * chunk_length_samples
161
+ duration_samples = rec.get_num_samples()
162
+
163
+ if need_motion_peaks:
164
+ for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, batch_length_samples)):
165
+ batch_end_samples = min(duration_samples, batch_start_samples + batch_length_samples)
166
+ res = dartsort.detect_for_motion(
167
+ recording=rec,
168
+ output_dir=ds_dir,
169
+ previous_sorting=subtraction_sorting,
170
+ hdf5_filename=f"motionthreshold_batch{batch_ix:05}.h5",
171
+ chunk_starts_samples=np.arange(batch_start_samples, batch_end_samples, chunk_length_samples),
172
+ motion_cfg=internal_cfg.motion_estimation_cfg,
173
+ computation_cfg=internal_cfg.motion_computation_cfg,
174
+ sampling_cfg=internal_cfg.peeler_sampling_cfg,
175
+ waveform_cfg=internal_cfg.waveform_cfg,
176
+ )
177
+ del res
178
+
179
+ # %% [markdown]
180
+ # Combine the output of those detections into one sorting, and check that together the
181
+ # batches covered the recording.
182
+
183
+ # %%
184
+ if need_motion_peaks and not (ds_dir / "motionthreshold.h5").exists():
185
+ motion_chunk_h5s = sorted(ds_dir.glob("motionthreshold_batch*.h5"))
186
+ dartsort.virtual_concatenate_sorting_h5s(motion_chunk_h5s, ds_dir / "motionthreshold.h5")
187
+ motion_sorting = dartsort.load(ds_dir / "motionthreshold.h5")
188
+ assert np.array_equal(
189
+ motion_sorting.load_dataset("chunk_starts_samples"),
190
+ np.arange(0, duration_samples, chunk_length_samples),
191
+ )
192
+ print(motion_sorting)
193
+
194
+ # %%
195
+ # !ls -lah {ds_dir}
196
+
197
+ # %% [markdown]
198
+ # ^ Notice all of the chunked motion files! But motionthreshold.h5 itself is tiny, because the concatenation is virtual.
199
+
200
+ # %% [markdown]
201
+ # # Resume: estimate motion, cluster, learn templates and the matching step classifier
202
+
203
+ # %%
204
+ dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_matching_models_only=True))
136
205
 
137
206
  # %%
138
207
  # !ls {ds_dir}
@@ -146,11 +215,8 @@ dartsort.dartsort(rec, ds_dir, cfg=cfg)
146
215
  # %% [markdown]
147
216
  # # Chunked template matching
148
217
  #
149
- # This runs template matching in 1-minute batches.
150
- #
151
- # Here, `chunk_starts_samples` slices the recording for us.
152
- # This might be better than using `.frame_slice` or `.time_slice` on the recording,
153
- # since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
218
+ # This runs template matching in 1-minute batches, with the same loop over chunks
219
+ # as the motion tracking's spike detection above.
154
220
  #
155
221
 
156
222
  # %%
@@ -186,7 +252,9 @@ for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, 60 * 3
186
252
  # which can be turned off for users who don't need them.
187
253
 
188
254
  # %%
189
- sorting = dartsort.concatenate_sortings([dartsort.load(h5) for h5 in sorted(ds_dir.glob("matching1_batch*.h5"))])
255
+ sorting = dartsort.concatenate_sortings(
256
+ [dartsort.load(h5) for h5 in sorted(ds_dir.glob("matching1_batch*.h5"))]
257
+ )
190
258
 
191
259
  # %%
192
260
  sorting = dartsort.cluster(
@@ -199,7 +267,7 @@ sorting = dartsort.cluster(
199
267
 
200
268
  # %%
201
269
  # if you want to keep this...
202
- sorting.save(ds_dir / 'dartsort_sorting.npz')
270
+ sorting.save(ds_dir / "dartsort_sorting.npz")
203
271
 
204
272
  # %% [markdown]
205
273
  # # Visualize
@@ -5,25 +5,26 @@
5
5
  # extension: .py
6
6
  # format_name: percent
7
7
  # format_version: '1.3'
8
- # jupytext_version: 1.19.3
8
+ # jupytext_version: 1.19.6
9
9
  # kernelspec:
10
- # display_name: Python [conda env:dart14]
10
+ # display_name: Python [conda env:a]
11
11
  # language: python
12
- # name: conda-env-dart14-py
12
+ # name: conda-env-a-py
13
13
  # ---
14
14
 
15
15
  # %%
16
16
  import matplotlib.pyplot as plt
17
17
 
18
+ # %%
19
+ import spikeinterface.full as si
20
+
18
21
  # %%
19
22
  # dandi-related
20
23
  from dandi.dandiapi import DandiAPIClient
21
- import fsspec
22
24
 
23
- # %%
24
- import spikeinterface.full as si
25
25
  import dartsort
26
26
  import dartsort.vis as dartvis
27
+
27
28
  dartvis.set_plt_style()
28
29
 
29
30
  # %% [markdown]
@@ -31,9 +32,7 @@ dartvis.set_plt_style()
31
32
 
32
33
  # %%
33
34
  # update this for your machine!
34
- experiment_path = dartsort.ensure_path(
35
- "~/scratch/ibl-dandi-example/", mkdir=True
36
- )
35
+ experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
37
36
 
38
37
  # %%
39
38
  # IBL BWM dandiset, example recording
@@ -71,7 +70,7 @@ else:
71
70
  print(rec0)
72
71
 
73
72
  # slice 10 minutes near the end
74
- #TODO better IBL sampling rate handling
73
+ # TODO better IBL sampling rate handling
75
74
  rec = rec0
76
75
  end_time = rec.get_end_time()
77
76
  start_frame = rec.time_to_sample_index(end_time - 30 * 60)
@@ -81,20 +80,19 @@ else:
81
80
  rec._sampling_frequency = 30_000
82
81
 
83
82
  rec = dartsort.preprocess(rec, strategy="ibllike")
84
- rec = rec.save_to_folder(preprocessing_path)
83
+ rec = rec.save(folder=preprocessing_path, format="binary")
85
84
 
86
85
  # %%
87
86
  plt.imshow(
88
87
  rec.get_traces(0, 10_000, 11_000).T,
89
- aspect='auto',
88
+ aspect="auto",
90
89
  vmin=-5,
91
90
  vmax=5,
92
- interpolation='nearest',
91
+ interpolation="nearest",
93
92
  )
94
- plt.colorbar(shrink=0.5, label='standardized voltage')
95
- plt.xlabel('time (samples')
96
- plt.ylabel('channels');
97
-
93
+ plt.colorbar(shrink=0.5, label="standardized voltage")
94
+ plt.xlabel("time (samples")
95
+ plt.ylabel("channels")
98
96
  # %% [markdown]
99
97
  # ## Run *dartsort*
100
98
 
@@ -106,6 +104,6 @@ ds_res = dartsort.dartsort(rec, ds_dir)
106
104
  # # Visualize
107
105
 
108
106
  # %%
109
- dartvis.scatter_spike_features(sorting=ds_res["sorting"]);
107
+ dartvis.scatter_spike_features(sorting=ds_res["sorting"])
110
108
 
111
109
  # %%
@@ -18,7 +18,6 @@ dependencies = [
18
18
  "dredge-ephys>=0.4.0",
19
19
  "h5py",
20
20
  "jaxtyping",
21
- "linear_operator",
22
21
  "numba",
23
22
  "numpy>=1.20;python_version<'3.13'",
24
23
  "numpy>=2.0.0;python_version>='3.13'",
@@ -63,6 +63,7 @@ from .util.data_util import (
63
63
  sorting_from_spikeinterface,
64
64
  try_get_denoising_pipeline,
65
65
  try_get_model_dir,
66
+ virtual_concatenate_sorting_h5s,
66
67
  )
67
68
  from .util.drift_util import registered_geometry
68
69
  from .util.internal_config import *
@@ -80,13 +81,14 @@ from .util.logging_util import (
80
81
  get_logger,
81
82
  set_log_level,
82
83
  )
84
+ from .util.main_util import motion_needs_peaks
83
85
  from .util.motion import (
84
86
  MotionInfo,
85
87
  detect_for_motion,
86
88
  get_motion_info,
87
89
  try_load_motion_info,
88
90
  )
89
- from .util.noise_util import EmbeddedNoise
91
+ from .util.noise_util import KroneckerNoise
90
92
  from .util.preprocess_util import preprocess
91
93
  from .util.py_util import databag, ensure_path
92
94
  from .util.waveform_util import full_channel_index, make_channel_index
@@ -243,7 +243,7 @@ class StableWaveformFeatures:
243
243
  # n, nc_extract
244
244
  channels: Tensor
245
245
  # n, rank, nc_extract
246
- features: Tensor
246
+ features: Tensor | np.ndarray
247
247
  neighborhoods: SpikeNeighborhoods
248
248
  erp: StableFeaturesInterpolator
249
249
 
@@ -292,6 +292,9 @@ class StableWaveformFeatures:
292
292
  trim_to_rank=clustering_features_cfg.feature_rank,
293
293
  params=clustering_features_cfg.interp_params.normalize(),
294
294
  device=computation_cfg.actual_device(),
295
+ store_on_disk=computation_cfg.low_memory,
296
+ tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
297
+ nan_to_num=True,
295
298
  )
296
299
  else:
297
300
  assert sorting.parent_h5_path is not None
@@ -308,13 +311,18 @@ class StableWaveformFeatures:
308
311
  trim_to_rank=clustering_features_cfg.feature_rank,
309
312
  params=clustering_features_cfg.interp_params.normalize(),
310
313
  device=computation_cfg.actual_device(),
314
+ store_on_disk=computation_cfg.low_memory,
315
+ tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
316
+ nan_to_num=True,
311
317
  )
312
318
 
313
- logger.dartsortdebug(f"StableWaveformFeatures {features.shape=}.")
319
+ logger.dartsortdebug(
320
+ f"StableWaveformFeatures {features.shape=} {computation_cfg.low_memory=}."
321
+ )
314
322
 
315
323
  return cls(
316
324
  channels=channels,
317
- features=torch.asarray(features),
325
+ features=features,
318
326
  neighborhoods=spike_neighborhoods,
319
327
  erp=erp,
320
328
  )
@@ -323,6 +331,17 @@ class StableWaveformFeatures:
323
331
  # -- helpers
324
332
 
325
333
 
334
+ def _read_ix_maybe_mmap(x: Tensor | np.ndarray, ix: Tensor | np.ndarray | slice) -> Tensor:
335
+ if isinstance(x, Tensor):
336
+ return x[ix]
337
+ if isinstance(ix, slice):
338
+ return torch.from_numpy(np.array(x[ix]))
339
+ if isinstance(ix, Tensor):
340
+ ix = ix.numpy(force=True)
341
+ assert ix.ndim == 1
342
+ return torch.from_numpy(x[ix])
343
+
344
+
326
345
  def _interpolate_main_channel_features(
327
346
  *,
328
347
  sorting: DARTsortSorting,