DARTsort 0.5.14__tar.gz → 0.5.15__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. {dartsort-0.5.14 → dartsort-0.5.15}/PKG-INFO +1 -1
  2. dartsort-0.5.15/notebook/ibl_dandi_example.py +111 -0
  3. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/agglomerate.py +28 -11
  4. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/density.py +1 -1
  5. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/mixture.py +21 -12
  6. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/config.py +3 -1
  7. dartsort-0.5.15/src/dartsort/detect/detect.py +185 -0
  8. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/peel_base.py +4 -1
  9. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/peel_lib.py +16 -32
  10. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/shaver.py +1 -8
  11. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/subtract.py +5 -9
  12. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/threshold.py +1 -9
  13. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/internal_config.py +0 -2
  14. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort.egg-info/PKG-INFO +1 -1
  15. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort.egg-info/SOURCES.txt +1 -0
  16. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_dartsort.py +2 -2
  17. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_data_util.py +1 -1
  18. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_detect.py +7 -15
  19. dartsort-0.5.14/src/dartsort/detect/detect.py +0 -408
  20. {dartsort-0.5.14 → dartsort-0.5.15}/.github/workflows/ci.yml +0 -0
  21. {dartsort-0.5.14 → dartsort-0.5.15}/.github/workflows/ci_windows.yml +0 -0
  22. {dartsort-0.5.14 → dartsort-0.5.15}/.github/workflows/deploy.yml +0 -0
  23. {dartsort-0.5.14 → dartsort-0.5.15}/.github/workflows/docs.yml +0 -0
  24. {dartsort-0.5.14 → dartsort-0.5.15}/.gitignore +0 -0
  25. {dartsort-0.5.14 → dartsort-0.5.15}/README.md +0 -0
  26. {dartsort-0.5.14 → dartsort-0.5.15}/docs/index.md +0 -0
  27. {dartsort-0.5.14 → dartsort-0.5.15}/docs/main_api.md +0 -0
  28. {dartsort-0.5.14 → dartsort-0.5.15}/docs/toolkit_api.md +0 -0
  29. {dartsort-0.5.14 → dartsort-0.5.15}/environment.yml +0 -0
  30. {dartsort-0.5.14 → dartsort-0.5.15}/notebook/.keep +0 -0
  31. {dartsort-0.5.14 → dartsort-0.5.15}/pyproject.toml +0 -0
  32. {dartsort-0.5.14 → dartsort-0.5.15}/setup.cfg +0 -0
  33. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/__init__.py +0 -0
  34. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/cli.py +0 -0
  35. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/__init__.py +0 -0
  36. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/cluster_util.py +0 -0
  37. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/clustering.py +0 -0
  38. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/clustering_features.py +0 -0
  39. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/forward_backward.py +0 -0
  40. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/kmeans.py +0 -0
  41. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/merge.py +0 -0
  42. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/modes.py +0 -0
  43. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/clustering/refine_util.py +0 -0
  44. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/detect/__init__.py +0 -0
  45. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/detect/detect_filters.py +0 -0
  46. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/evaluate/__init__.py +0 -0
  47. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/evaluate/analysis.py +0 -0
  48. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/evaluate/comparison.py +0 -0
  49. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/evaluate/hybrid_util.py +0 -0
  50. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  51. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  52. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/evaluate/simkit.py +0 -0
  53. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/evaluate/simlib.py +0 -0
  54. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/localize/__init__.py +0 -0
  55. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/localize/localize_torch.py +0 -0
  56. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/localize/localize_util.py +0 -0
  57. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/main.py +0 -0
  58. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/__init__.py +0 -0
  59. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/grab.py +0 -0
  60. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/matching.py +0 -0
  61. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/matching_util/__init__.py +0 -0
  62. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  63. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/matching_util/drifty.py +0 -0
  64. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  65. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  66. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  67. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/peel/reduction_template.py +0 -0
  68. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  69. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/pretrained/detect_np1.pt +0 -0
  70. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/pretrained/detect_np2.pt +0 -0
  71. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  72. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  73. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  74. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  75. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  76. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  77. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/templates/__init__.py +0 -0
  78. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/templates/get_templates.py +0 -0
  79. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/templates/postprocess_util.py +0 -0
  80. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/templates/realignment.py +0 -0
  81. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/templates/template_util.py +0 -0
  82. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/templates/templates.py +0 -0
  83. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/templates/templib.py +0 -0
  84. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/__init__.py +0 -0
  85. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  86. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/all_transformers.py +0 -0
  87. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/amortized_localization.py +0 -0
  88. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/amplitudes.py +0 -0
  89. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/decollider.py +0 -0
  90. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/denoising_scorer.py +0 -0
  91. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/enforce_decrease.py +0 -0
  92. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/fixed_prop.py +0 -0
  93. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/interp.py +0 -0
  94. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/localize.py +0 -0
  95. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/matching_denoiser.py +0 -0
  96. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/mixture_classifier.py +0 -0
  97. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/pipeline.py +0 -0
  98. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/reduction.py +0 -0
  99. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  100. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/supervised_denoiser.py +0 -0
  101. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/temporal_pca.py +0 -0
  102. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/transform_base.py +0 -0
  103. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/transform/whiten.py +0 -0
  104. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/__init__.py +0 -0
  105. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/cli_util.py +0 -0
  106. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/data_util.py +0 -0
  107. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/drift_util.py +0 -0
  108. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/interpolation_util.py +0 -0
  109. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/job_util.py +0 -0
  110. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/logging_util.py +0 -0
  111. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/main_util.py +0 -0
  112. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/more_operators.py +0 -0
  113. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/motion.py +0 -0
  114. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/multiprocessing_util.py +0 -0
  115. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/nn_util.py +0 -0
  116. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/noise_util.py +0 -0
  117. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/peel_util.py +0 -0
  118. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/preprocess_util.py +0 -0
  119. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/py_util.py +0 -0
  120. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/registration_util.py +0 -0
  121. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/residual_util.py +0 -0
  122. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/sparse_util.py +0 -0
  123. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/spikeio.py +0 -0
  124. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/spiketorch.py +0 -0
  125. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/testing_util/__init__.py +0 -0
  126. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  127. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  128. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/torch_optimization_util.py +0 -0
  129. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/torch_util.py +0 -0
  130. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/util/waveform_util.py +0 -0
  131. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/__init__.py +0 -0
  132. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/analysis_plots.py +0 -0
  133. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/colors.py +0 -0
  134. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/gt.py +0 -0
  135. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/layout.py +0 -0
  136. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/metrics_util.py +0 -0
  137. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/mixture.py +0 -0
  138. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/recanim.py +0 -0
  139. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/scatterplots.py +0 -0
  140. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/sorting.py +0 -0
  141. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/unit.py +0 -0
  142. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/unit_comparison.py +0 -0
  143. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/versus.py +0 -0
  144. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/vismain.py +0 -0
  145. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort/vis/waveforms.py +0 -0
  146. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort.egg-info/dependency_links.txt +0 -0
  147. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort.egg-info/entry_points.txt +0 -0
  148. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort.egg-info/requires.txt +0 -0
  149. {dartsort-0.5.14 → dartsort-0.5.15}/src/dartsort.egg-info/top_level.txt +0 -0
  150. {dartsort-0.5.14 → dartsort-0.5.15}/tests/conftest.py +0 -0
  151. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_agglomerate.py +0 -0
  152. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_alignment.py +0 -0
  153. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_cluster_util.py +0 -0
  154. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_clustering.py +0 -0
  155. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_config.py +0 -0
  156. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_density.py +0 -0
  157. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_drift_util.py +0 -0
  158. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_drifty_matching.py +0 -0
  159. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_enforce_decrease.py +0 -0
  160. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_grab_and_featurize.py +0 -0
  161. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_kmeans.py +0 -0
  162. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_linalg.py +0 -0
  163. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_localize_torch.py +0 -0
  164. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_matching.py +0 -0
  165. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_mixture_classifier.py +0 -0
  166. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_modeling.py +0 -0
  167. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_simkit.py +0 -0
  168. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_spiketorch.py +0 -0
  169. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_subtract.py +0 -0
  170. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_templates.py +0 -0
  171. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_threshold.py +0 -0
  172. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_torch_optimization_util.py +0 -0
  173. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_transform.py +0 -0
  174. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_util.py +0 -0
  175. {dartsort-0.5.14 → dartsort-0.5.15}/tests/test_waveform_util.py +0 -0
  176. {dartsort-0.5.14 → dartsort-0.5.15}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.14
3
+ Version: 0.5.15
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -0,0 +1,111 @@
1
+ # ---
2
+ # jupyter:
3
+ # jupytext:
4
+ # text_representation:
5
+ # extension: .py
6
+ # format_name: percent
7
+ # format_version: '1.3'
8
+ # jupytext_version: 1.19.3
9
+ # kernelspec:
10
+ # display_name: Python [conda env:dart14]
11
+ # language: python
12
+ # name: conda-env-dart14-py
13
+ # ---
14
+
15
+ # %%
16
+ import matplotlib.pyplot as plt
17
+
18
+ # %%
19
+ # dandi-related
20
+ from dandi.dandiapi import DandiAPIClient
21
+ import fsspec
22
+
23
+ # %%
24
+ import spikeinterface.full as si
25
+ import dartsort
26
+ import dartsort.vis as dartvis
27
+ dartvis.set_plt_style()
28
+
29
+ # %% [markdown]
30
+ # ### Paths / config
31
+
32
+ # %%
33
+ # update this for your machine!
34
+ experiment_path = dartsort.ensure_path(
35
+ "~/scratch/ibl-dandi-example/", mkdir=True
36
+ )
37
+
38
+ # %%
39
+ # IBL BWM dandiset, example recording
40
+ dandiset_id = "000409"
41
+ asset_path = "sub-CSH-ZAD-026/sub-CSH-ZAD-026_ses-15763234-d21e-491f-a01b-1238eb96d389_desc-raw_ecephys.nwb"
42
+
43
+ # %% [markdown]
44
+ # ### Preprocess and save a 10 minute snippet of data
45
+
46
+ # %% [markdown]
47
+ # Pre-saving the preprocessing like this is optional. You can also set the
48
+ # preprocessing flag of the `DARTsortUserConfig()` to the same strategy (`"ibllike"`)
49
+ # or one of the others. In that case, you can also set `copy_recording_to_tmpdir` to
50
+ # reduce preprocessing overhead. Here, I'm pre-saving it just to visualize and in
51
+ # case the user wants to experiment with the cached preprocessed recording.
52
+
53
+ # %%
54
+ preprocessing_path = experiment_path / "preprocessed_recording"
55
+ if preprocessing_path.exists():
56
+ rec = si.read_binary_folder(preprocessing_path)
57
+ else:
58
+ si.set_global_job_kwargs(n_jobs=8, pool_engine="thread")
59
+
60
+ # stream from DANDI with SpikeInterface
61
+ # start by finding the s3 asset URL
62
+ with DandiAPIClient() as client:
63
+ asset = client.get_dandiset(dandiset_id, "draft").get_asset_by_path(asset_path)
64
+ s3_url = asset.get_content_url(follow_redirects=1, strip_query=True)
65
+ rec0 = si.read_nwb_recording(
66
+ s3_url,
67
+ stream_mode="fsspec",
68
+ electrical_series_path="acquisition/ElectricalSeriesProbe00AP",
69
+ )
70
+ print("Original recording:")
71
+ print(rec0)
72
+
73
+ # slice 10 minutes near the end
74
+ #TODO better IBL sampling rate handling
75
+ rec = rec0
76
+ end_time = rec.get_end_time()
77
+ start_frame = rec.time_to_sample_index(end_time - 30 * 60)
78
+ end_frame = start_frame + 10 * 60 * 30_000
79
+ rec = rec.frame_slice(start_frame, end_frame)
80
+ rec.reset_times()
81
+ rec._sampling_frequency = 30_000
82
+
83
+ rec = dartsort.preprocess(rec, strategy="ibllike")
84
+ rec = rec.save_to_folder(preprocessing_path)
85
+
86
+ # %%
87
+ plt.imshow(
88
+ rec.get_traces(0, 10_000, 11_000).T,
89
+ aspect='auto',
90
+ vmin=-5,
91
+ vmax=5,
92
+ interpolation='nearest',
93
+ )
94
+ plt.colorbar(shrink=0.5, label='standardized voltage')
95
+ plt.xlabel('time (samples')
96
+ plt.ylabel('channels');
97
+
98
+ # %% [markdown]
99
+ # ## Run *dartsort*
100
+
101
+ # %%
102
+ ds_dir = experiment_path / "dartsort"
103
+ ds_res = dartsort.dartsort(rec, ds_dir)
104
+
105
+ # %% [markdown]
106
+ # # Visualize
107
+
108
+ # %%
109
+ dartvis.scatter_spike_features(sorting=ds_res["sorting"]);
110
+
111
+ # %%
@@ -71,7 +71,7 @@ def agglomerate(
71
71
  template_merge_cfg = refinement_cfg.template_merge_cfg
72
72
 
73
73
  if template_data is None:
74
- sorting = sorting.flatten()
74
+ sorting = sorting.flatten(include_gmm_properties=True)
75
75
 
76
76
  tdist = template_distances(
77
77
  sorting=sorting,
@@ -751,6 +751,8 @@ def combine_gmm_scores(
751
751
  responsibilities = getattr(sorting, f"{old_prefix}_responsibilities", None)
752
752
  logliks = getattr(sorting, f"{old_prefix}_log_liks", None)
753
753
 
754
+ assert not hasattr(sorting, f"{new_prefix}_responsibilities")
755
+
754
756
  havec = candidates is not None
755
757
  haver = responsibilities is not None
756
758
  havel = logliks is not None
@@ -780,10 +782,9 @@ def combine_gmm_scores(
780
782
  assert _maxdiff <= 1e-3, _maxdiff
781
783
  assert np.greater_equal(np.isneginf(logliks[:, :-1]), candidates == -1).all()
782
784
  if sorting.labels is not None:
783
- assert np.all(
784
- np.logical_or(
785
- sorting.labels < 0, sorting.labels == new_ids[candidates[:, 0]]
786
- )
785
+ not_noise = np.flatnonzero(sorting.labels >= 0)
786
+ assert np.array_equal(
787
+ sorting.labels[not_noise], new_ids[candidates[not_noise, 0]]
787
788
  )
788
789
 
789
790
  # two steps: first merge, then sort
@@ -839,20 +840,24 @@ def _combine_loop(
839
840
  mergedl: np.ndarray,
840
841
  ):
841
842
  for s in numba.prange(cand.shape[0]): # ty: ignore
842
- rcand = cand[s]
843
+ spike_cand = cand[s]
843
844
  for j in range(cand.shape[1] - 1):
844
- ncandj = rcand[j]
845
- if ncandj < 0 or new_id_counts[ncandj] <= 1:
845
+ spike_candj = spike_cand[j]
846
+
847
+ # noise or not a merge
848
+ if spike_candj < 0 or new_id_counts[spike_candj] <= 1:
846
849
  continue
847
850
 
848
- eq_ncandj = rcand[j + 1 :] == ncandj
849
- if eq_ncandj.sum() < 1:
851
+ # what later indices are equal to me?
852
+ eq_spike_candj = spike_cand[j + 1 :] == spike_candj
853
+ if eq_spike_candj.sum() < 1:
850
854
  continue
851
855
 
856
+ # loop through and combine liks/resps, and -1 out the cands
852
857
  rsum = mergedr[s, j]
853
858
  lsum = mergedl[s, j]
854
859
  for i, k in enumerate(range(j + 1, cand.shape[1])):
855
- if not eq_ncandj[i]:
860
+ if not eq_spike_candj[i]:
856
861
  continue
857
862
  cand[s, k] = -1
858
863
 
@@ -867,6 +872,18 @@ def _combine_loop(
867
872
  mergedr[s, j] = rsum
868
873
  mergedl[s, j] = lsum
869
874
 
875
+ # vacuum into noise component
876
+ # this is partly to handle stuff that was missed before getting here
877
+ # from flatten, for example
878
+ for j in range(cand.shape[1]):
879
+ if 0 <= spike_cand[j] < new_id_counts.shape[0]:
880
+ continue
881
+ rsj = mergedr[s, j]
882
+ if rsj == 0:
883
+ continue
884
+ mergedr[s, -1] += rsj
885
+ mergedr[s, j] = 0.0
886
+
870
887
 
871
888
  def deduplicate_spikes(
872
889
  sorting: DARTsortSorting,
@@ -85,7 +85,7 @@ def get_smoothed_density(
85
85
  # figure out what bandwidths we'll be working on
86
86
 
87
87
  if do_ramp := bool(sigma_low):
88
- min_sigma = min(sigma, sigma_low) # type: ignore
88
+ min_sigma = min(sigma, sigma_low) # type: ignore # ty: ignore[x]
89
89
  else:
90
90
  min_sigma = sigma
91
91
 
@@ -607,13 +607,13 @@ class LUTParams(BModule):
607
607
  self.n_lut = n0 = n_lut_new
608
608
 
609
609
  self.b.muo.resize_(n0, *self.b.muo.shape[1:])
610
- self.b.muo.fill_(0.0)
610
+ self.b.muo.zero_()
611
611
  self.b.Linvmuo.resize_(n0, *self.b.Linvmuo.shape[1:])
612
- self.b.Linvmuo.fill_(0.0)
612
+ self.b.Linvmuo.zero_()
613
613
  self.b.CmoCooinvmuo.resize_(n0, *self.b.CmoCooinvmuo.shape[1:])
614
- self.b.CmoCooinvmuo.fill_(0.0)
614
+ self.b.CmoCooinvmuo.zero_()
615
615
  self.b.constplogdet.resize_(n0, *self.b.constplogdet.shape[1:])
616
- self.b.constplogdet.fill_(0.0)
616
+ self.b.constplogdet.zero_()
617
617
  if not self.signal_rank:
618
618
  return
619
619
  assert self.TWoCooinvsqrt is not None
@@ -621,13 +621,13 @@ class LUTParams(BModule):
621
621
  assert self.Tpad is not None
622
622
  assert self.wburyroot is not None
623
623
  self.b.TWoCooinvsqrt.resize_(n0, *self.b.TWoCooinvsqrt.shape[1:])
624
- self.b.TWoCooinvsqrt.fill_(0.0)
624
+ self.b.TWoCooinvsqrt.zero_()
625
625
  self.b.TWoCooinvmuo.resize_(n0, *self.b.TWoCooinvmuo.shape[1:])
626
- self.b.TWoCooinvmuo.fill_(0.0)
626
+ self.b.TWoCooinvmuo.zero_()
627
627
  self.b.Tpad.resize_(n0, *self.b.Tpad.shape[1:])
628
- self.b.Tpad.fill_(0.0)
628
+ self.b.Tpad.zero_()
629
629
  self.b.wburyroot.resize_(n0, *self.b.wburyroot.shape[1:])
630
- self.b.wburyroot.fill_(0.0)
630
+ self.b.wburyroot.zero_()
631
631
 
632
632
  def check(self):
633
633
  muo_f = self.b.muo.isfinite().all()
@@ -1342,6 +1342,7 @@ class BatchedSpikeData:
1342
1342
  allow_new_units: bool,
1343
1343
  allow_uncovered: bool = False,
1344
1344
  max_steps: int | None = None,
1345
+ next_label: int | None = None,
1345
1346
  ):
1346
1347
  self.update_adjacency(n_units=None)
1347
1348
  assert self.candidates is not None
@@ -1356,6 +1357,7 @@ class BatchedSpikeData:
1356
1357
  ensure_coverage_only=True,
1357
1358
  allow_new_units=allow_new_units,
1358
1359
  allow_uncovered=allow_uncovered,
1360
+ next_label=next_label,
1359
1361
  max_steps=max_steps if max_steps is not None else self.coverage_steps,
1360
1362
  )
1361
1363
  assert res is None
@@ -1864,6 +1866,8 @@ class TruncatedSpikeData(BatchedSpikeData):
1864
1866
  ids_new = remapping.mapping.unique()
1865
1867
  ids_new = ids_new[ids_new >= 0]
1866
1868
  n_units_new = ids_new.shape[0]
1869
+ _n_units_new = ids_new.amax() + 1
1870
+ assert _n_units_new == n_units_new
1867
1871
  assert self.candidates is not None
1868
1872
  assert n_units_new <= n_units_orig
1869
1873
 
@@ -1871,7 +1875,8 @@ class TruncatedSpikeData(BatchedSpikeData):
1871
1875
  assert distances.shape[0] == distances.shape[1]
1872
1876
  assert distances.shape[0] == n_units_new
1873
1877
  assert distances.shape[0] <= n_units_orig
1874
- assert remapping.mapping.max() + 1 == distances.shape[0]
1878
+ assert remapping.mapping.amax() + 1 == distances.shape[0]
1879
+ assert _n_units_new == distances.shape[0]
1875
1880
 
1876
1881
  self._update_sizes_from_n_units(n_units_new)
1877
1882
 
@@ -1900,6 +1905,7 @@ class TruncatedSpikeData(BatchedSpikeData):
1900
1905
  allow_new_units=not allow_uncovered,
1901
1906
  allow_uncovered=allow_uncovered,
1902
1907
  max_steps=0 if allow_uncovered else None,
1908
+ next_label=n_units_new,
1903
1909
  )
1904
1910
  if allow_uncovered:
1905
1911
  assert n_new == 0
@@ -6025,8 +6031,8 @@ def coincidence_matrix(
6025
6031
  yy = y[pos_ii]
6026
6032
 
6027
6033
  if pnoid:
6028
- assert xx.max() < nx
6029
- assert yy.max() < ny
6034
+ assert xx.max() < nx, f"{xx.max()} {nx}"
6035
+ assert yy.max() < ny, f"{yy.max()} {ny}"
6030
6036
  assert xx.min() >= 0
6031
6037
  assert yy.min() >= 0
6032
6038
 
@@ -6152,6 +6158,7 @@ def _fill_blank_labels(
6152
6158
  ensure_coverage_only: bool = False,
6153
6159
  allow_uncovered: bool = False,
6154
6160
  allow_new_units: bool = False,
6161
+ next_label: int | None = None,
6155
6162
  max_steps=2,
6156
6163
  ):
6157
6164
  assert neighborhood_ids.shape == labels.shape
@@ -6170,6 +6177,8 @@ def _fill_blank_labels(
6170
6177
  # explore as needed, so make those probs tiny.
6171
6178
  keep_same_adj = un_adj.sum(0).amin().cpu().item() > 0
6172
6179
  n_new_units = 0
6180
+ if next_label is None:
6181
+ next_label = un_adj.shape[0]
6173
6182
  if keep_same_adj:
6174
6183
  adj = un_adj
6175
6184
  else:
@@ -6210,7 +6219,7 @@ def _fill_blank_labels(
6210
6219
  n_spikes += in_nid.numel()
6211
6220
  if pnoid:
6212
6221
  assert (labels[in_nid] < 0).all()
6213
- labels[in_nid] = un_adj.shape[0] + neighb_component_ids[j]
6222
+ labels[in_nid] = next_label + neighb_component_ids[j]
6214
6223
  if pnoid:
6215
6224
  Knew = labels.max() + 1 + int((labels < 0).any())
6216
6225
  assert labels.unique().shape[0] <= Knew
@@ -205,6 +205,9 @@ class DARTsortUserConfig:
205
205
  """Upsampling of templates during matching to allow for temporal aliasing of waveforms."""
206
206
 
207
207
  # -- final merge step
208
+ agg_kind: Literal["none", "template_distance", "qda"] = "qda"
209
+ """Final distance or GMM-based merge type."""
210
+
208
211
  spikeinterface_merge_preset: str | Literal["none"] | None = None
209
212
  """Call out to SpikeInterface's auto_merge() for a final merge using timing / RP information.
210
213
  Setting this is slightly different' from calling auto_merge() externally, since the internal
@@ -369,7 +372,6 @@ class DeveloperConfig(DARTsortUserConfig):
369
372
  tpca_from_templates: bool = True
370
373
 
371
374
  # agglomeration
372
- agg_kind: Literal["none", "template_distance", "qda"] = "qda"
373
375
  agg_qda_max_template_distance: float = 0.6
374
376
  agg_no_qda_template_distance: float = 0.3
375
377
  agg_qda_linkage: Literal["single", "complete"] = "single"
@@ -0,0 +1,185 @@
1
+ from typing import Literal
2
+
3
+ import torch
4
+ import torch.nn.functional as F
5
+ from torch import Tensor
6
+
7
+ # holds values used for deduplicating identical peaks
8
+ _salt = {}
9
+ _pepper = {}
10
+
11
+
12
+ def detect_and_deduplicate(
13
+ traces: Tensor,
14
+ threshold: float,
15
+ peak_sign: Literal["pos", "neg", "both"] = "neg",
16
+ relative_peak_radius=5,
17
+ peak_channel_index: Tensor | None = None,
18
+ dedup_temporal_radius=11,
19
+ dedup_channel_index: torch.Tensor | None = None,
20
+ trough_priority: float | None = None,
21
+ batch_size=1024,
22
+ remove_exact_duplicates=True,
23
+ detection_mask: Tensor | None = None,
24
+ exclude_edges=True,
25
+ return_energies=False,
26
+ dedup_salt_eps=1e-4,
27
+ ):
28
+ """Detect and deduplicate peaks
29
+
30
+ torch-based peak detection and deduplication, relying
31
+ on max pooling and scatter operations
32
+
33
+ Parameters
34
+ ----------
35
+ traces : time by channels tensor
36
+ threshold : float
37
+ dedup_channel_index : channels by n_neighbors tensor
38
+ Channel neighbors index. (See waveform_util for
39
+ more on this format.) If supplied, peaks are kept
40
+ only when they are the largest among their neighbors
41
+ as described by this array
42
+ peak_sign : one of "neg", "pos", "both"
43
+ relative_peak_radius : int
44
+ How many temporal neighbors must you be taller than
45
+ to be considered a peak?
46
+ dedup_temporal_radius : int
47
+ Only the largest peak within this sliding radius
48
+ will be kept
49
+
50
+ Returns
51
+ -------
52
+ times, chans : tensors of shape (n_peaks,)
53
+ peak times in samples relative to start of traces, along
54
+ with corresponding channels
55
+ """
56
+ T, C = traces.shape
57
+ all_peaks = torch.zeros_like(traces, dtype=torch.bool)
58
+ will_dedup = bool(dedup_temporal_radius) or dedup_channel_index is not None
59
+ pad = relative_peak_radius + dedup_temporal_radius
60
+ batch_size = batch_size - 2 * pad
61
+ assert batch_size > 0
62
+
63
+ if dedup_channel_index is not None and remove_exact_duplicates:
64
+ key = (traces.device.type, traces.device.index, C)
65
+ if key in _salt:
66
+ dedup_chans_salt = _salt[key]
67
+ else:
68
+ # favor earlier channels when deduplicating identical values (at float16 prec)
69
+ dedup_chans_salt = torch.linspace(
70
+ dedup_salt_eps, 0.0, steps=C, device=traces.device
71
+ )
72
+ _salt[key] = dedup_chans_salt = dedup_chans_salt.unsqueeze(1)
73
+ else:
74
+ dedup_chans_salt = None
75
+
76
+ if dedup_temporal_radius and remove_exact_duplicates:
77
+ slen = 2 * dedup_temporal_radius
78
+ nrep = (T // slen) + 1
79
+ Tp = slen * nrep
80
+ key = (traces.device.type, traces.device.index, Tp, dedup_temporal_radius)
81
+ if key in _pepper:
82
+ dedup_time_salt = _pepper[key]
83
+ else:
84
+ # favor earlier times when deduplicating identical values (at float16 prec)
85
+ dedup_time_salt = torch.linspace(
86
+ dedup_salt_eps, 0.0, steps=slen, device=traces.device
87
+ )
88
+ dedup_time_salt = dedup_time_salt.repeat(nrep)
89
+ _pepper[key] = dedup_time_salt = dedup_time_salt
90
+ else:
91
+ dedup_time_salt = None
92
+
93
+ for i0 in range(0, T, batch_size):
94
+ i1 = min(T, i0 + batch_size)
95
+ i00 = max(0, i0 - pad)
96
+ istart = i0 - i00
97
+ i11 = min(T, i1 + pad)
98
+ iend = istart + (i1 - i0)
99
+
100
+ # life is easier in channels-major here
101
+ X = F.pad(traces[i00:i11].T, (0, 0, 0, 1))
102
+
103
+ # data for threshold and deduplication criteria
104
+ if peak_sign == "neg":
105
+ Xdd = Xth = X.neg_()
106
+ elif peak_sign == "pos":
107
+ Xdd = Xth = X
108
+ elif peak_sign == "both":
109
+ if trough_priority:
110
+ # deduplication takes trough priority into account.
111
+ Xdd = F.leaky_relu(X, negative_slope=-trough_priority)
112
+ Xth = X.abs_()
113
+ else:
114
+ Xdd = Xth = X.abs_()
115
+ else:
116
+ assert False
117
+
118
+ # -- detect peaks
119
+ detect = Xth[:-1] > threshold
120
+ peak = _is_extreme(
121
+ Xdd,
122
+ dt=relative_peak_radius,
123
+ neighbors=peak_channel_index,
124
+ )
125
+ detect = detect.logical_and_(peak)
126
+ tmp = peak
127
+ del peak
128
+
129
+ # check if deduping
130
+ if not will_dedup:
131
+ all_peaks[i0:i1] = detect[:, istart:iend].T
132
+ continue
133
+
134
+ # -- deduplicate peaks
135
+ if dedup_chans_salt is not None:
136
+ Xdd[:-1] += dedup_chans_salt
137
+ if dedup_time_salt is not None:
138
+ Xdd[:-1] += dedup_time_salt[i00:i11]
139
+ mask_out = torch.logical_not(detect, out=tmp)
140
+ if detection_mask is not None:
141
+ mask_out.logical_and_(detection_mask[i00:i11].T)
142
+ Xdd[:-1].masked_fill_(mask_out, 0.0)
143
+
144
+ # no-threshold max pool for deduplication
145
+ dedup = _is_extreme(
146
+ Xdd, dt=dedup_temporal_radius, neighbors=dedup_channel_index, out=tmp
147
+ )
148
+ all_peaks[i0:i1] = detect.logical_and_(dedup)[:, istart:iend].T
149
+
150
+ if exclude_edges:
151
+ all_peaks[0].zero_()
152
+ all_peaks[-1].zero_()
153
+
154
+ times, chans = all_peaks.nonzero(as_tuple=True)
155
+ if return_energies:
156
+ return times, chans, traces[times, chans].abs_()
157
+ else:
158
+ return times, chans
159
+
160
+
161
+ def _is_extreme(
162
+ X: Tensor,
163
+ dt=5,
164
+ neighbors: Tensor | None = None,
165
+ out=None,
166
+ ):
167
+ if neighbors is not None:
168
+ # CT -> C, n_neighbors, T
169
+ Xneighb = X[neighbors]
170
+ Xmax = Xneighb.amax(dim=1)
171
+ else:
172
+ Xmax = X[:-1]
173
+ Xmax = F.max_pool1d(
174
+ Xmax[None],
175
+ stride=(1,),
176
+ kernel_size=(2 * dt + 1,),
177
+ padding=(dt,),
178
+ )
179
+ Xmax = Xmax[0]
180
+
181
+ # if max pool made you grow, or if thresholding made you grow,
182
+ # then you were not a peak
183
+ peak = torch.ge(X[:-1], Xmax, out=out)
184
+
185
+ return peak
@@ -801,7 +801,10 @@ class BasePeeler(BModule):
801
801
  waveforms_dataset_name="peeled_waveforms_fit",
802
802
  **fixed_properties,
803
803
  )
804
- assert getrefcount(waveforms) <= 2
804
+ if getrefcount(waveforms) > 2:
805
+ logger.warn(f"Fit waveforms had {getrefcount(waveforms)=}")
806
+ for obj in gc.get_referrers(waveforms):
807
+ logger.warn(f"{repr(obj)}: {str(obj)}")
805
808
  featurization_pipeline = featurization_pipeline.to("cpu")
806
809
  self.featurization_pipeline = featurization_pipeline
807
810
  finally: