DARTsort 0.5.34__tar.gz → 0.5.35__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. {dartsort-0.5.34 → dartsort-0.5.35}/PKG-INFO +2 -2
  2. {dartsort-0.5.34 → dartsort-0.5.35}/notebook/fit_and_batch_matching.py +44 -13
  3. {dartsort-0.5.34 → dartsort-0.5.35}/pyproject.toml +1 -1
  4. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/__init__.py +2 -0
  5. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/clustering/agglomerate.py +106 -87
  6. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/clustering/cluster_util.py +21 -6
  7. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/config.py +16 -2
  8. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/main.py +86 -14
  9. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/peel_base.py +14 -2
  10. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/data_util.py +186 -41
  11. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/internal_config.py +4 -1
  12. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/main_util.py +7 -5
  13. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/motion.py +8 -10
  14. dartsort-0.5.35/src/dartsort/util/registration_util.py +188 -0
  15. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/scatterplots.py +6 -0
  16. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort.egg-info/PKG-INFO +2 -2
  17. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort.egg-info/requires.txt +1 -1
  18. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_agglomerate.py +23 -3
  19. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_dartsort.py +13 -1
  20. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_data_util.py +72 -0
  21. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_simkit.py +2 -1
  22. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_threshold.py +31 -0
  23. dartsort-0.5.34/src/dartsort/util/registration_util.py +0 -103
  24. {dartsort-0.5.34 → dartsort-0.5.35}/.github/workflows/ci.yml +0 -0
  25. {dartsort-0.5.34 → dartsort-0.5.35}/.github/workflows/ci_windows.yml +0 -0
  26. {dartsort-0.5.34 → dartsort-0.5.35}/.github/workflows/deploy.yml +0 -0
  27. {dartsort-0.5.34 → dartsort-0.5.35}/.github/workflows/docs.yml +0 -0
  28. {dartsort-0.5.34 → dartsort-0.5.35}/.gitignore +0 -0
  29. {dartsort-0.5.34 → dartsort-0.5.35}/README.md +0 -0
  30. {dartsort-0.5.34 → dartsort-0.5.35}/docs/index.md +0 -0
  31. {dartsort-0.5.34 → dartsort-0.5.35}/docs/main_api.md +0 -0
  32. {dartsort-0.5.34 → dartsort-0.5.35}/docs/toolkit_api.md +0 -0
  33. {dartsort-0.5.34 → dartsort-0.5.35}/environment.yml +0 -0
  34. {dartsort-0.5.34 → dartsort-0.5.35}/notebook/.keep +0 -0
  35. {dartsort-0.5.34 → dartsort-0.5.35}/notebook/ibl_dandi_example.py +0 -0
  36. {dartsort-0.5.34 → dartsort-0.5.35}/setup.cfg +0 -0
  37. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/__main__.py +0 -0
  38. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/cli.py +0 -0
  39. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/clustering/__init__.py +0 -0
  40. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/clustering/clustering.py +0 -0
  41. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/clustering/clustering_features.py +0 -0
  42. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/clustering/density.py +0 -0
  43. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/clustering/forward_backward.py +0 -0
  44. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/clustering/kmeans.py +0 -0
  45. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/clustering/merge.py +0 -0
  46. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/clustering/mixture.py +0 -0
  47. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/clustering/refine_util.py +0 -0
  48. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/detect/__init__.py +0 -0
  49. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/detect/detect.py +0 -0
  50. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/evaluate/__init__.py +0 -0
  51. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/evaluate/analysis.py +0 -0
  52. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/evaluate/comparison.py +0 -0
  53. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/evaluate/hybrid_util.py +0 -0
  54. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/evaluate/hybridkit.py +0 -0
  55. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  56. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  57. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/evaluate/simkit.py +0 -0
  58. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/evaluate/simlib.py +0 -0
  59. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/localize/__init__.py +0 -0
  60. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/localize/localize_torch.py +0 -0
  61. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/localize/localize_util.py +0 -0
  62. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/__init__.py +0 -0
  63. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/grab.py +0 -0
  64. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/matching.py +0 -0
  65. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/matching_util/__init__.py +0 -0
  66. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  67. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/matching_util/drifty.py +0 -0
  68. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  69. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  70. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  71. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/peel_lib.py +0 -0
  72. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/reduction_template.py +0 -0
  73. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/shaver.py +0 -0
  74. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/subtract.py +0 -0
  75. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/subtract_util.py +0 -0
  76. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/peel/threshold.py +0 -0
  77. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  78. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/pretrained/detect_np1.pt +0 -0
  79. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/pretrained/detect_np2.pt +0 -0
  80. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  81. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  82. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  83. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  84. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  85. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  86. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/templates/__init__.py +0 -0
  87. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/templates/get_templates.py +0 -0
  88. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/templates/postprocess_util.py +0 -0
  89. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/templates/realignment.py +0 -0
  90. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/templates/template_util.py +0 -0
  91. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/templates/templates.py +0 -0
  92. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/templates/templib.py +0 -0
  93. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/__init__.py +0 -0
  94. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  95. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/all_transformers.py +0 -0
  96. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/amortized_localization.py +0 -0
  97. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/amplitudes.py +0 -0
  98. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/decollider.py +0 -0
  99. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/denoising_scorer.py +0 -0
  100. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/enforce_decrease.py +0 -0
  101. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/fixed_prop.py +0 -0
  102. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/interp.py +0 -0
  103. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/localize.py +0 -0
  104. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/matched_filter_net.py +0 -0
  105. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/matching_denoiser.py +0 -0
  106. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/mixture_classifier.py +0 -0
  107. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/pipeline.py +0 -0
  108. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/reduction.py +0 -0
  109. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  110. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/supervised_denoiser.py +0 -0
  111. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/temporal_pca.py +0 -0
  112. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/transform_base.py +0 -0
  113. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/vq_matched_filter.py +0 -0
  114. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/transform/whiten.py +0 -0
  115. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/__init__.py +0 -0
  116. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/cli_util.py +0 -0
  117. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/drift_util.py +0 -0
  118. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/interpolation_util.py +0 -0
  119. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/job_util.py +0 -0
  120. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/logging_util.py +0 -0
  121. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/multiprocessing_util.py +0 -0
  122. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/nn_util.py +0 -0
  123. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/noise_util.py +0 -0
  124. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/peel_util.py +0 -0
  125. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/preprocess_util.py +0 -0
  126. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/py_util.py +0 -0
  127. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/residual_util.py +0 -0
  128. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/sparse_util.py +0 -0
  129. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/spikeio.py +0 -0
  130. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/spiketorch.py +0 -0
  131. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/testing_util/__init__.py +0 -0
  132. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  133. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  134. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/torch_optimization_util.py +0 -0
  135. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/torch_util.py +0 -0
  136. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/util/waveform_util.py +0 -0
  137. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/__init__.py +0 -0
  138. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/analysis_plots.py +0 -0
  139. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/colors.py +0 -0
  140. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/gt.py +0 -0
  141. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/layout.py +0 -0
  142. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/metrics_util.py +0 -0
  143. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/mixture.py +0 -0
  144. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/recanim.py +0 -0
  145. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/sorting.py +0 -0
  146. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/unit.py +0 -0
  147. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/unit_comparison.py +0 -0
  148. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/versus.py +0 -0
  149. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/vismain.py +0 -0
  150. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort/vis/waveforms.py +0 -0
  151. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort.egg-info/SOURCES.txt +0 -0
  152. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort.egg-info/dependency_links.txt +0 -0
  153. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort.egg-info/entry_points.txt +0 -0
  154. {dartsort-0.5.34 → dartsort-0.5.35}/src/dartsort.egg-info/top_level.txt +0 -0
  155. {dartsort-0.5.34 → dartsort-0.5.35}/tests/conftest.py +0 -0
  156. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_alignment.py +0 -0
  157. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_cluster_util.py +0 -0
  158. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_clustering.py +0 -0
  159. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_clustering_features.py +0 -0
  160. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_config.py +0 -0
  161. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_density.py +0 -0
  162. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_detect.py +0 -0
  163. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_drift_util.py +0 -0
  164. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_drifty_matching.py +0 -0
  165. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_enforce_decrease.py +0 -0
  166. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_grab_and_featurize.py +0 -0
  167. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_hybridkit.py +0 -0
  168. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_kmeans.py +0 -0
  169. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_localize_torch.py +0 -0
  170. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_matching.py +0 -0
  171. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_mixture_classifier.py +0 -0
  172. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_modeling.py +0 -0
  173. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_multiprocessing_util.py +0 -0
  174. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_spiketorch.py +0 -0
  175. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_subtract.py +0 -0
  176. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_templates.py +0 -0
  177. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_torch_optimization_util.py +0 -0
  178. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_transform.py +0 -0
  179. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_util.py +0 -0
  180. {dartsort-0.5.34 → dartsort-0.5.35}/tests/test_waveform_util.py +0 -0
  181. {dartsort-0.5.34 → dartsort-0.5.35}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.34
3
+ Version: 0.5.35
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -10,7 +10,7 @@ Classifier: License :: OSI Approved :: MIT License
10
10
  Classifier: Operating System :: OS Independent
11
11
  Requires-Python: >=3.12
12
12
  Description-Content-Type: text/markdown
13
- Requires-Dist: dredge-ephys>=0.4.0
13
+ Requires-Dist: dredge-ephys>=0.5.0
14
14
  Requires-Dist: h5py
15
15
  Requires-Dist: jaxtyping
16
16
  Requires-Dist: numba
@@ -20,6 +20,11 @@
20
20
  #
21
21
  # This pipeline is intended for users with very long recordings who want more control over the matching step.
22
22
  #
23
+ # To that end, this also showcases *dartsort*'s `low_memory` flag. In that setting, *dartsort* falls back to
24
+ # memmaps in certain situations or uses streaming algorithms to avoid ever allocating any array containing
25
+ # spikes across the whole dataset, although it is allowed to store per-spike arrays in stages where the data
26
+ # is subsampled.
27
+ #
23
28
  # Since the matching step is the only one which passes over the whole recording, long recordings can benefit
24
29
  # from additional parallelism there beyond *dartsort*'s built-in single- or multi-GPU parallelism.
25
30
  # In this notebook, the matching step is run "chunked", but just in a for loop without parallelism.
@@ -33,6 +38,7 @@
33
38
  # %%
34
39
  import dataclasses
35
40
 
41
+ import h5py
36
42
  import matplotlib.pyplot as plt
37
43
  import numpy as np
38
44
  import spikeinterface.full as si
@@ -53,6 +59,9 @@ experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=Tru
53
59
  # %%
54
60
  ds_dir = experiment_path / "dartsort-chunked"
55
61
 
62
+ # %%
63
+ # import shutil; shutil.rmtree(ds_dir)
64
+
56
65
  # %%
57
66
  # IBL BWM dandiset, example recording
58
67
  dandiset_id = "000409"
@@ -116,8 +125,8 @@ plt.ylabel("channels")
116
125
  # # Configuration
117
126
 
118
127
  # %%
119
- # use defaults, except for the pipeline change
120
- cfg = dartsort.DARTsortUserConfig()
128
+ # low_memory mode! otherwise, defaults are fine.
129
+ cfg = dartsort.DARTsortUserConfig(low_memory=True)
121
130
 
122
131
  # %%
123
132
  # process into an internal config object for later use
@@ -125,6 +134,14 @@ internal_cfg = dartsort.to_internal_config(cfg)
125
134
 
126
135
  # %% [markdown]
127
136
  # # Initial spike detection and motion tracking model fits
137
+ #
138
+ # The first stage should set `fit_motion_models_only`, since motion tracking requires extracting
139
+ # spikes across the full dataset and should therefore be chunked in this setting. That chunking
140
+ # happens in the next section.
141
+ #
142
+ # If you're not doing motion tracking, you can still keep all of this stuff; later stages will
143
+ # resume from the work that was finished here. Or, you could equivalently start from the next
144
+ # call to `dartsort()` below the "Resume: estimate..." heading.
128
145
 
129
146
  # %%
130
147
  dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_motion_models_only=True))
@@ -184,12 +201,12 @@ if need_motion_peaks:
184
201
  if need_motion_peaks and not (ds_dir / "motionthreshold.h5").exists():
185
202
  motion_chunk_h5s = sorted(ds_dir.glob("motionthreshold_batch*.h5"))
186
203
  dartsort.virtual_concatenate_sorting_h5s(motion_chunk_h5s, ds_dir / "motionthreshold.h5")
187
- motion_sorting = dartsort.load(ds_dir / "motionthreshold.h5")
188
- assert np.array_equal(
189
- motion_sorting.load_dataset("chunk_starts_samples"),
190
- np.arange(0, duration_samples, chunk_length_samples),
191
- )
192
- print(motion_sorting)
204
+ with h5py.File(ds_dir / "motionthreshold.h5", "r", locking=False) as h5:
205
+ assert np.array_equal(
206
+ h5["chunk_starts_samples"][:],
207
+ np.arange(0, duration_samples, chunk_length_samples),
208
+ )
209
+ print(f"{h5['times_samples'].shape[0]} spikes for motion tracking")
193
210
 
194
211
  # %%
195
212
  # !ls -lah {ds_dir}
@@ -245,16 +262,26 @@ for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, 60 * 3
245
262
  #
246
263
  # Users can change this. If you don't want to merge, set `postprocessing="clean"` in the configuration
247
264
  # section above. In that case, you could run the cleaning step on each chunk before concatenating: the
248
- # merge needs to agree across the batched matching output, but the output of cleaning is the same when
249
- # run batchwise or on the full result.
265
+ # merge needs to agree across the batched matching output, but the output of cleaning is approximately the
266
+ # same when run batchwise or on the full concatenation, except near chunk boundaries.
250
267
  #
251
268
  # Here, extra features (amplitudes, localizations) are loaded by the default behavior of `dartsort.load`,
252
269
  # which can be turned off for users who don't need them.
253
270
 
254
271
  # %%
255
- sorting = dartsort.concatenate_sortings(
256
- [dartsort.load(h5) for h5 in sorted(ds_dir.glob("matching1_batch*.h5"))]
257
- )
272
+ matching_h5 = ds_dir / "matching1.h5"
273
+ if not matching_h5.exists():
274
+ dartsort.virtual_concatenate_sorting_h5s(
275
+ sorted(ds_dir.glob("matching1_batch*.h5")), matching_h5
276
+ )
277
+ if internal_cfg.computation_cfg.low_memory:
278
+ dartsort.pack_peeling_hdf5_to_npz(matching_h5, ds_dir / "matching1.npz")
279
+ sorting = dartsort.load(ds_dir / "matching1.npz", low_memory=True)
280
+ sorting = dartsort.with_labels_in_memmap(
281
+ sorting, ds_dir / "matching1_labels_tmp_memmap.npy"
282
+ )
283
+ else:
284
+ sorting = dartsort.load(matching_h5)
258
285
 
259
286
  # %%
260
287
  sorting = dartsort.cluster(
@@ -268,6 +295,10 @@ sorting = dartsort.cluster(
268
295
  # %%
269
296
  # if you want to keep this...
270
297
  sorting.save(ds_dir / "dartsort_sorting.npz")
298
+ if internal_cfg.computation_cfg.low_memory:
299
+ sorting = dartsort.load(ds_dir / "dartsort_sorting.npz", low_memory=True)
300
+ (ds_dir / "matching1_labels_tmp_memmap.npy").unlink()
301
+ (ds_dir / "matching1.npz").unlink()
271
302
 
272
303
  # %% [markdown]
273
304
  # # Visualize
@@ -15,7 +15,7 @@ classifiers = [
15
15
  "Operating System :: OS Independent",
16
16
  ]
17
17
  dependencies = [
18
- "dredge-ephys>=0.4.0",
18
+ "dredge-ephys>=0.5.0",
19
19
  "h5py",
20
20
  "jaxtyping",
21
21
  "numba",
@@ -60,10 +60,12 @@ from .util.data_util import (
60
60
  get_tpca,
61
61
  load,
62
62
  load_stored_tsvd,
63
+ pack_peeling_hdf5_to_npz,
63
64
  sorting_from_spikeinterface,
64
65
  try_get_denoising_pipeline,
65
66
  try_get_model_dir,
66
67
  virtual_concatenate_sorting_h5s,
68
+ with_labels_in_memmap,
67
69
  )
68
70
  from .util.drift_util import registered_geometry
69
71
  from .util.internal_config import *
@@ -14,8 +14,6 @@ from ..templates.templates import TemplateData
14
14
  from ..util.data_util import (
15
15
  DARTsortSorting,
16
16
  apply_label_remapping_in_place,
17
- count_not_sorted,
18
- pos_int_unique_and_counts,
19
17
  )
20
18
  from ..util.internal_config import (
21
19
  ComputationConfig,
@@ -1122,6 +1120,7 @@ def clean_final_sorting(
1122
1120
  merge_mapping: np.ndarray | None = None,
1123
1121
  score_by=("gmm_log_liks", "scores"),
1124
1122
  in_place: bool = True,
1123
+ batch_size: int = 2**20,
1125
1124
  ) -> tuple[DARTsortSorting, np.ndarray]:
1126
1125
  """Deduplicate, flatten, depth-order
1127
1126
 
@@ -1148,8 +1147,12 @@ def clean_final_sorting(
1148
1147
  )
1149
1148
  dedup_ms = -1.0
1150
1149
 
1151
- sorting = deduplicate_spikes(sorting, dedup_ms, in_place=in_place)
1152
- sorting, reorder = reorder_by_depth(sorting, motion=motion, in_place=in_place)
1150
+ sorting = deduplicate_spikes(
1151
+ sorting, dedup_ms, in_place=in_place, batch_size=batch_size
1152
+ )
1153
+ sorting, reorder = reorder_by_depth(
1154
+ sorting, motion=motion, in_place=in_place, batch_size=batch_size
1155
+ )
1153
1156
  if merge_mapping is None:
1154
1157
  return sorting, reorder
1155
1158
  return sorting, reorder[np.unique(merge_mapping, return_inverse=True)[1]]
@@ -1160,6 +1163,7 @@ def deduplicate_spikes(
1160
1163
  radius_ms: float = -1.0,
1161
1164
  score_by=("gmm_log_liks", "scores"),
1162
1165
  in_place: bool = False,
1166
+ batch_size: int = 2**20,
1163
1167
  ) -> DARTsortSorting:
1164
1168
  if radius_ms < 0 or sorting.labels is None:
1165
1169
  return sorting
@@ -1171,110 +1175,125 @@ def deduplicate_spikes(
1171
1175
  assert radius_samples >= 0
1172
1176
 
1173
1177
  new_labels = sorting.labels if in_place else sorting.labels.copy()
1174
- scores = None
1175
- for sck in score_by:
1176
- if not sorting.has_dataset(sck):
1177
- continue
1178
- sl = (slice(None), 0) if sck.endswith("log_liks") else ()
1179
- scores = sorting.load_dataset(sck, sl=sl)
1180
- if scores is not None:
1181
- logger.dartsortdebug(f"deduplicate by score {sck}")
1178
+ assert new_labels.flags.writeable
1179
+ for score_name in score_by:
1180
+ if sorting.has_dataset(score_name):
1182
1181
  break
1183
- if scores is None:
1184
- raise ValueError(f"sorting had none of {score_by}.")
1185
- if scores.ndim >= 2:
1186
- scores = scores[:, 0]
1187
- assert scores.ndim == 1
1188
- assert scores.shape == new_labels.shape
1189
-
1190
- # handle unsorted times
1191
- if count_not_sorted(sorting.times_samples) > 0:
1192
- tsort = np.argsort(sorting.times_samples, kind="stable")
1193
- labels_by_time = new_labels[tsort]
1194
- times_samples = sorting.times_samples[tsort]
1195
- scores = scores[tsort]
1196
1182
  else:
1197
- tsort = None
1198
- labels_by_time = new_labels
1199
- times_samples = sorting.times_samples
1183
+ raise ValueError(f"sorting had none of {score_by}.")
1184
+ logger.dartsortdebug(f"deduplicate by score {score_name}")
1185
+ score_col = (0,) if score_name.endswith("log_liks") else ()
1186
+
1187
+ times_samples = sorting.times_samples
1188
+ n = new_labels.shape[0]
1189
+ assert times_samples.shape == (n,)
1190
+ if sorting.get_mask_indices() is not None:
1191
+ batch_size = max(1, n)
1192
+ batch_starts = range(0, n, batch_size)
1193
+ batch_tmin = np.array(
1194
+ [times_samples[i0 : i0 + batch_size].min() for i0 in batch_starts]
1195
+ )
1196
+ later_tmin = np.minimum.accumulate(batch_tmin[::-1])[::-1]
1197
+ later_tmin = np.r_[later_tmin[1:], np.iinfo(times_samples.dtype).max]
1200
1198
 
1201
- unit_ids, _, _ = pos_int_unique_and_counts(labels_by_time)
1202
1199
  ndrop = 0
1203
- unit_mask_tmp = np.empty(labels_by_time.shape, dtype=bool)
1204
- for unit_id in unit_ids:
1205
- in_unit = np.flatnonzero(np.equal(labels_by_time, unit_id, out=unit_mask_tmp))
1206
- if in_unit.size <= 1:
1207
- continue
1208
- t = times_samples[in_unit]
1209
- dt = np.diff(t)
1210
- if dt.min() > radius_samples:
1211
- continue
1212
- discard = dedup_unit(t, dt, scores[in_unit], radius_samples)
1200
+ prev_unfinished = np.zeros(0, dtype=np.intp)
1201
+ for b, i0 in enumerate(batch_starts):
1202
+ i1 = min(n, i0 + batch_size)
1203
+ labeled = new_labels[i0:i1] >= 0
1204
+ ix = np.concatenate([prev_unfinished, i0 + np.flatnonzero(labeled)])
1205
+ scores = sorting.load_dataset(score_name, sl=(slice(i0, i1), *score_col))
1206
+ scores = scores[labeled]
1207
+ del labeled
1208
+ if prev_unfinished.size:
1209
+ prev_scores = sorting.load_dataset(
1210
+ score_name, sl=(prev_unfinished, *score_col)
1211
+ )
1212
+ scores = np.concatenate([prev_scores, scores])
1213
+ del prev_scores
1214
+ if scores.ndim >= 2:
1215
+ scores = scores[:, 0]
1216
+ assert scores.shape == ix.shape
1217
+
1218
+ t = times_samples[ix]
1219
+ order = np.lexsort((ix, t, new_labels[ix]))
1220
+ ix = ix[order]
1221
+ t = t[order]
1222
+ scores = scores[order]
1223
+ del order
1224
+ unit_bounds = np.flatnonzero(np.diff(new_labels[ix])) + 1
1225
+ unit_bounds = np.r_[0, unit_bounds, ix.size]
1226
+
1227
+ discard = np.zeros(ix.shape, dtype=bool)
1228
+ unfinished = np.zeros(ix.shape, dtype=bool)
1229
+ _dedup_batch(
1230
+ t, scores, unit_bounds, later_tmin[b], radius_samples, discard, unfinished
1231
+ )
1213
1232
  ndrop += discard.sum()
1214
- labels_by_time[in_unit[discard]] = -1
1233
+ new_labels[ix[discard]] = -1
1234
+ prev_unfinished = np.sort(ix[unfinished])
1235
+ assert not prev_unfinished.size
1215
1236
 
1216
1237
  logger.dartsortdebug(f"drop {ndrop}/{len(sorting)} isi violator spikes")
1217
- if tsort is not None:
1218
- new_labels[tsort] = labels_by_time
1219
-
1220
1238
  return sorting.ephemeral_replace(labels=new_labels)
1221
1239
 
1222
1240
 
1223
- def dedup_unit(
1224
- t: np.ndarray, dt: np.ndarray, scores: np.ndarray, radius: int
1225
- ) -> np.ndarray:
1226
- """Deduplicate a single unit's spike train."""
1227
- discard = np.zeros(t.shape, dtype=bool)
1228
- _dedup_unit_loop(dt, scores, radius, discard)
1229
- return discard
1230
-
1231
-
1232
1241
  @numba.njit(nogil=True)
1233
1242
  def _dedup_unit_loop(
1234
- dt: np.ndarray, scores: np.ndarray, radius: int, discard: np.ndarray
1243
+ t: np.ndarray, scores: np.ndarray, radius: int, discard: np.ndarray
1235
1244
  ):
1236
- n = scores.shape[0]
1245
+ n = t.shape[0]
1237
1246
  i0 = 0
1238
1247
  while i0 < n - 1:
1239
- if dt[i0] > radius:
1248
+ if t[i0 + 1] - t[i0] > radius:
1240
1249
  i0 += 1
1241
1250
  continue
1242
1251
 
1243
1252
  i1 = i0 + 1
1244
- while i1 < n - 1 and dt[i1] <= radius:
1253
+ while i1 < n - 1 and t[i1 + 1] - t[i1] <= radius:
1245
1254
  i1 += 1
1246
1255
 
1247
1256
  # now, i0:i1 + 1 is a slice of violators
1248
- # discard until all valid
1249
- score_slice = scores[i0 : i1 + 1]
1250
- dt_slice = dt[i0:i1]
1251
- order = np.argsort(score_slice)
1252
- # putting the -1 there ensures at least one spike is kept
1253
- # that case is only relevant when 0 in dt_slice
1254
- for oo in order[:-1]:
1255
- # discard spike i0 + oo
1257
+ order = np.argsort(-scores[i0 : i1 + 1], kind="mergesort")
1258
+ for oo in order:
1256
1259
  ii = i0 + oo
1257
- discard[ii] = True
1258
-
1259
- # figure out isi after bridging the gap, handle edges
1260
- bridge_isi = 0.0
1261
- if ii < n - 1:
1262
- bridge_isi += dt[ii]
1263
- else:
1264
- bridge_isi = np.inf
1265
- if ii > 0:
1266
- bridge_isi += dt[ii - 1]
1267
- else:
1268
- bridge_isi = np.inf
1269
-
1270
- # update dt with bridge isi
1271
- if ii < n - 1:
1272
- dt[ii] = bridge_isi
1273
- if ii > 0:
1274
- dt[ii - 1] = bridge_isi
1275
-
1276
- # check if done
1277
- if dt_slice.min() > radius:
1278
- break
1260
+ if discard[ii]:
1261
+ continue
1262
+ j = ii
1263
+ while j > i0 and t[ii] - t[j - 1] <= radius:
1264
+ j -= 1
1265
+ discard[j] = True
1266
+ j = ii
1267
+ while j < i1 and t[j + 1] - t[ii] <= radius:
1268
+ j += 1
1269
+ discard[j] = True
1279
1270
 
1280
- i0 = i1
1271
+ i0 = i1 + 1
1272
+
1273
+
1274
+ @numba.njit(nogil=True)
1275
+ def _dedup_batch(
1276
+ t: np.ndarray,
1277
+ scores: np.ndarray,
1278
+ unit_bounds: np.ndarray,
1279
+ later_tmin: int,
1280
+ radius: int,
1281
+ discard: np.ndarray,
1282
+ unfinished: np.ndarray,
1283
+ ):
1284
+ for jj in range(unit_bounds.shape[0] - 1):
1285
+ j0 = unit_bounds[jj]
1286
+ j1 = unit_bounds[jj + 1]
1287
+ batch_end = j0 + np.searchsorted(t[j0:j1], later_tmin - radius)
1288
+ # walk back so that we end before any runs which could cross to the next chunk
1289
+ while (
1290
+ batch_end > j0
1291
+ and batch_end < j1
1292
+ and t[batch_end] - t[batch_end - 1] <= radius
1293
+ ):
1294
+ batch_end -= 1
1295
+ unfinished[batch_end:j1] = True
1296
+
1297
+ _dedup_unit_loop(
1298
+ t[j0:batch_end], scores[j0:batch_end], radius, discard[j0:batch_end]
1299
+ )
@@ -11,8 +11,8 @@ from ..util.data_util import (
11
11
  DARTsortSorting,
12
12
  apply_label_remapping_in_place,
13
13
  count_not_sorted,
14
- mean_by_label_1d,
15
14
  pos_int_unique_and_counts,
15
+ welford_by_label_1d,
16
16
  yield_masked_chunks,
17
17
  )
18
18
  from ..util.logging_util import get_logger
@@ -74,7 +74,10 @@ def apply_reclustering(
74
74
  else:
75
75
  assert unit_snrs is not None
76
76
  times_samples = sorting.times_samples
77
- times_updated = times_samples if in_place else times_samples.copy()
77
+ if in_place and times_samples.flags.writeable:
78
+ times_updated = times_samples
79
+ else:
80
+ times_updated = times_samples.copy()
78
81
  apply_time_shifts(
79
82
  times_updated,
80
83
  sorting.labels,
@@ -375,6 +378,7 @@ def reorder_by_depth(
375
378
  geom: np.ndarray | None = None,
376
379
  centroids: np.ndarray | None = None,
377
380
  in_place: bool = False,
381
+ batch_size: int = 2**20,
378
382
  ) -> tuple[DARTsortSorting, np.ndarray]:
379
383
  """Reorder cluster labels so that centroid depth is increasing
380
384
 
@@ -409,10 +413,21 @@ def reorder_by_depth(
409
413
  centroids = w @ geom[:, 1]
410
414
 
411
415
  if centroids is None:
412
- z = sorting.load_dataset("point_source_localizations", sl=(slice(None), 2))
413
- if motion is not None:
414
- z = motion.correct_s(sorting.times_seconds, z)
415
- centroids = mean_by_label_1d(sorting, z)
416
+ n = len(sorting)
417
+ if sorting.get_mask_indices() is not None:
418
+ batch_size = max(1, n)
419
+ centroids = np.zeros(sorting.n_units)
420
+ counts = np.zeros(sorting.n_units)
421
+ for i0 in range(0, n, batch_size):
422
+ i1 = min(n, i0 + batch_size)
423
+ z = sorting.load_dataset(
424
+ "point_source_localizations", sl=(slice(i0, i1), 2)
425
+ )
426
+ if motion is not None:
427
+ t = sorting.load_dataset("times_seconds", sl=slice(i0, i1))
428
+ z = motion.correct_s(t, z)
429
+ welford_by_label_1d(sorting.labels[i0:i1], z, centroids, counts)
430
+ assert counts.min(initial=1) > 0
416
431
 
417
432
  labels = sorting.labels if in_place else sorting.labels.copy()
418
433
  reorder = np.argsort(np.argsort(centroids, kind="stable"), kind="stable")
@@ -139,7 +139,10 @@ class DARTsortUserConfig:
139
139
  tmpdir_parent: str | None = None
140
140
  """Control where tmpdirs are created."""
141
141
  low_memory: bool = False
142
- """Best effort at not using tons of RAM. If you notice memory problems, please get in touch!"""
142
+ """Best effort at not using tons of RAM. If you notice memory problems, please get in
143
+ touch! Note that process pools may be used in some places here, so if you're running
144
+ dartsort in a script with low_memory=True you'll need to wrap it in an
145
+ `if __name__ == '__main__'` block."""
143
146
  save_intermediates: bool = False
144
147
  """Store all spike features from intermediate steps (for debugging)"""
145
148
 
@@ -267,9 +270,20 @@ class DARTsortUserConfig:
267
270
  """Motion bins farther than this from the local median will be replaced by interpolation."""
268
271
  median_neighborhood_bins: int = 51
269
272
  motion_batching_mode: MotionBatchingMode = "full"
270
- """'full' or 'online'. Online helps with long recordings."""
273
+ """'full', 'coarse_guided_online', 'online'.
274
+
275
+ If you have very long recordings (hundreds-thousands of hours), "coarse_guided_online" is
276
+ recommended: "full" cannot run at that scale, coarse-guided is a good happy medium.
277
+ """
271
278
  motion_chunk_len_s: Annotated[float, Field(gt=0)] = 3000.0
272
279
  """Chunk length in seconds for online motion tracking"""
280
+ motion_coarse_bin_s: Annotated[float, Field(gt=0)] = 600.0
281
+ """Coarse bin length for 'coarse_guided_online'"""
282
+ motion_coarse_max_dt_s: Annotated[float, Field(gt=0)] | None = None
283
+ """max_dt_s of the coarse solve -- infinite (None) by default"""
284
+ motion_fine_max_disp_um: Annotated[float, Field(gt=0)] | None = None
285
+ """Fine timescale search range in coarse_guided_online motion. This is how much
286
+ wiggle is modeled after correcting for the coarse (long timescale) motion."""
273
287
 
274
288
  # -- pipeline control parameters for specific use cases
275
289
  fit_motion_models_only: bool = False