DARTsort 0.5.14__tar.gz → 0.5.16__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. {dartsort-0.5.14 → dartsort-0.5.16}/.github/workflows/ci.yml +2 -2
  2. {dartsort-0.5.14 → dartsort-0.5.16}/PKG-INFO +1 -1
  3. dartsort-0.5.16/notebook/ibl_dandi_example.py +111 -0
  4. {dartsort-0.5.14 → dartsort-0.5.16}/pyproject.toml +4 -2
  5. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/agglomerate.py +88 -40
  6. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/cluster_util.py +9 -7
  7. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/density.py +1 -1
  8. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/kmeans.py +7 -7
  9. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/mixture.py +158 -77
  10. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/config.py +3 -1
  11. dartsort-0.5.16/src/dartsort/detect/detect.py +190 -0
  12. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/comparison.py +100 -0
  13. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/simkit.py +24 -17
  14. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching.py +11 -5
  15. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/compressed_upsampled.py +2 -1
  16. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/drifty.py +9 -11
  17. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/matching_base.py +4 -3
  18. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/pairwise.py +1 -3
  19. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/peel_base.py +5 -2
  20. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/peel_lib.py +16 -32
  21. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/shaver.py +1 -8
  22. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/subtract.py +5 -9
  23. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/threshold.py +1 -9
  24. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/decollider.py +6 -5
  25. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/mixture_classifier.py +5 -4
  26. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/pipeline.py +4 -0
  27. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/data_util.py +73 -28
  28. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/internal_config.py +17 -10
  29. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/noise_util.py +2 -2
  30. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/py_util.py +5 -1
  31. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/spiketorch.py +7 -6
  32. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/testing_util/matching_debug_util.py +3 -2
  33. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/vismain.py +31 -17
  34. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/PKG-INFO +1 -1
  35. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/SOURCES.txt +1 -0
  36. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_clustering.py +1 -2
  37. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_dartsort.py +2 -2
  38. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_data_util.py +1 -1
  39. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_detect.py +7 -15
  40. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_modeling.py +13 -11
  41. dartsort-0.5.14/src/dartsort/detect/detect.py +0 -408
  42. {dartsort-0.5.14 → dartsort-0.5.16}/.github/workflows/ci_windows.yml +0 -0
  43. {dartsort-0.5.14 → dartsort-0.5.16}/.github/workflows/deploy.yml +0 -0
  44. {dartsort-0.5.14 → dartsort-0.5.16}/.github/workflows/docs.yml +0 -0
  45. {dartsort-0.5.14 → dartsort-0.5.16}/.gitignore +0 -0
  46. {dartsort-0.5.14 → dartsort-0.5.16}/README.md +0 -0
  47. {dartsort-0.5.14 → dartsort-0.5.16}/docs/index.md +0 -0
  48. {dartsort-0.5.14 → dartsort-0.5.16}/docs/main_api.md +0 -0
  49. {dartsort-0.5.14 → dartsort-0.5.16}/docs/toolkit_api.md +0 -0
  50. {dartsort-0.5.14 → dartsort-0.5.16}/environment.yml +0 -0
  51. {dartsort-0.5.14 → dartsort-0.5.16}/notebook/.keep +0 -0
  52. {dartsort-0.5.14 → dartsort-0.5.16}/setup.cfg +0 -0
  53. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/__init__.py +0 -0
  54. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/cli.py +0 -0
  55. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/__init__.py +0 -0
  56. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/clustering.py +0 -0
  57. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/clustering_features.py +0 -0
  58. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/forward_backward.py +0 -0
  59. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/merge.py +0 -0
  60. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/modes.py +0 -0
  61. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/refine_util.py +0 -0
  62. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/detect/__init__.py +0 -0
  63. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/detect/detect_filters.py +0 -0
  64. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/__init__.py +0 -0
  65. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/analysis.py +0 -0
  66. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/hybrid_util.py +0 -0
  67. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  68. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  69. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/simlib.py +0 -0
  70. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/localize/__init__.py +0 -0
  71. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/localize/localize_torch.py +0 -0
  72. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/localize/localize_util.py +0 -0
  73. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/main.py +0 -0
  74. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/__init__.py +0 -0
  75. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/grab.py +0 -0
  76. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/__init__.py +0 -0
  77. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  78. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/reduction_template.py +0 -0
  79. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  80. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/detect_np1.pt +0 -0
  81. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/detect_np2.pt +0 -0
  82. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  83. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  84. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  85. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  86. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  87. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  88. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/__init__.py +0 -0
  89. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/get_templates.py +0 -0
  90. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/postprocess_util.py +0 -0
  91. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/realignment.py +0 -0
  92. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/template_util.py +0 -0
  93. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/templates.py +0 -0
  94. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/templib.py +0 -0
  95. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/__init__.py +0 -0
  96. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  97. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/all_transformers.py +0 -0
  98. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/amortized_localization.py +0 -0
  99. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/amplitudes.py +0 -0
  100. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/denoising_scorer.py +0 -0
  101. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/enforce_decrease.py +0 -0
  102. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/fixed_prop.py +0 -0
  103. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/interp.py +0 -0
  104. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/localize.py +0 -0
  105. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/matching_denoiser.py +0 -0
  106. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/reduction.py +0 -0
  107. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  108. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/supervised_denoiser.py +0 -0
  109. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/temporal_pca.py +0 -0
  110. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/transform_base.py +0 -0
  111. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/whiten.py +0 -0
  112. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/__init__.py +0 -0
  113. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/cli_util.py +0 -0
  114. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/drift_util.py +0 -0
  115. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/interpolation_util.py +0 -0
  116. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/job_util.py +0 -0
  117. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/logging_util.py +0 -0
  118. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/main_util.py +0 -0
  119. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/more_operators.py +0 -0
  120. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/motion.py +0 -0
  121. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/multiprocessing_util.py +0 -0
  122. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/nn_util.py +0 -0
  123. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/peel_util.py +0 -0
  124. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/preprocess_util.py +0 -0
  125. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/registration_util.py +0 -0
  126. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/residual_util.py +0 -0
  127. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/sparse_util.py +0 -0
  128. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/spikeio.py +0 -0
  129. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/testing_util/__init__.py +0 -0
  130. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  131. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/torch_optimization_util.py +0 -0
  132. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/torch_util.py +0 -0
  133. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/waveform_util.py +0 -0
  134. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/__init__.py +0 -0
  135. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/analysis_plots.py +0 -0
  136. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/colors.py +0 -0
  137. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/gt.py +0 -0
  138. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/layout.py +0 -0
  139. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/metrics_util.py +0 -0
  140. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/mixture.py +0 -0
  141. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/recanim.py +0 -0
  142. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/scatterplots.py +0 -0
  143. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/sorting.py +0 -0
  144. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/unit.py +0 -0
  145. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/unit_comparison.py +0 -0
  146. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/versus.py +0 -0
  147. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/waveforms.py +0 -0
  148. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/dependency_links.txt +0 -0
  149. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/entry_points.txt +0 -0
  150. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/requires.txt +0 -0
  151. {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/top_level.txt +0 -0
  152. {dartsort-0.5.14 → dartsort-0.5.16}/tests/conftest.py +0 -0
  153. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_agglomerate.py +0 -0
  154. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_alignment.py +0 -0
  155. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_cluster_util.py +0 -0
  156. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_config.py +0 -0
  157. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_density.py +0 -0
  158. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_drift_util.py +0 -0
  159. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_drifty_matching.py +0 -0
  160. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_enforce_decrease.py +0 -0
  161. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_grab_and_featurize.py +0 -0
  162. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_kmeans.py +0 -0
  163. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_linalg.py +0 -0
  164. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_localize_torch.py +0 -0
  165. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_matching.py +0 -0
  166. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_mixture_classifier.py +0 -0
  167. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_simkit.py +0 -0
  168. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_spiketorch.py +0 -0
  169. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_subtract.py +0 -0
  170. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_templates.py +0 -0
  171. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_threshold.py +0 -0
  172. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_torch_optimization_util.py +0 -0
  173. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_transform.py +0 -0
  174. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_util.py +0 -0
  175. {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_waveform_util.py +0 -0
  176. {dartsort-0.5.14 → dartsort-0.5.16}/zensical.toml +0 -0
@@ -6,8 +6,8 @@ on:
6
6
  - main
7
7
  - develop
8
8
  pull_request:
9
- branches:
10
- - "*"
9
+ branches-ignore:
10
+ - develop
11
11
  workflow_dispatch:
12
12
 
13
13
  jobs:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.14
3
+ Version: 0.5.16
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -0,0 +1,111 @@
1
+ # ---
2
+ # jupyter:
3
+ # jupytext:
4
+ # text_representation:
5
+ # extension: .py
6
+ # format_name: percent
7
+ # format_version: '1.3'
8
+ # jupytext_version: 1.19.3
9
+ # kernelspec:
10
+ # display_name: Python [conda env:dart14]
11
+ # language: python
12
+ # name: conda-env-dart14-py
13
+ # ---
14
+
15
+ # %%
16
+ import matplotlib.pyplot as plt
17
+
18
+ # %%
19
+ # dandi-related
20
+ from dandi.dandiapi import DandiAPIClient
21
+ import fsspec
22
+
23
+ # %%
24
+ import spikeinterface.full as si
25
+ import dartsort
26
+ import dartsort.vis as dartvis
27
+ dartvis.set_plt_style()
28
+
29
+ # %% [markdown]
30
+ # ### Paths / config
31
+
32
+ # %%
33
+ # update this for your machine!
34
+ experiment_path = dartsort.ensure_path(
35
+ "~/scratch/ibl-dandi-example/", mkdir=True
36
+ )
37
+
38
+ # %%
39
+ # IBL BWM dandiset, example recording
40
+ dandiset_id = "000409"
41
+ asset_path = "sub-CSH-ZAD-026/sub-CSH-ZAD-026_ses-15763234-d21e-491f-a01b-1238eb96d389_desc-raw_ecephys.nwb"
42
+
43
+ # %% [markdown]
44
+ # ### Preprocess and save a 10 minute snippet of data
45
+
46
+ # %% [markdown]
47
+ # Pre-saving the preprocessing like this is optional. You can also set the
48
+ # preprocessing flag of the `DARTsortUserConfig()` to the same strategy (`"ibllike"`)
49
+ # or one of the others. In that case, you can also set `copy_recording_to_tmpdir` to
50
+ # reduce preprocessing overhead. Here, I'm pre-saving it just to visualize and in
51
+ # case the user wants to experiment with the cached preprocessed recording.
52
+
53
+ # %%
54
+ preprocessing_path = experiment_path / "preprocessed_recording"
55
+ if preprocessing_path.exists():
56
+ rec = si.read_binary_folder(preprocessing_path)
57
+ else:
58
+ si.set_global_job_kwargs(n_jobs=8, pool_engine="thread")
59
+
60
+ # stream from DANDI with SpikeInterface
61
+ # start by finding the s3 asset URL
62
+ with DandiAPIClient() as client:
63
+ asset = client.get_dandiset(dandiset_id, "draft").get_asset_by_path(asset_path)
64
+ s3_url = asset.get_content_url(follow_redirects=1, strip_query=True)
65
+ rec0 = si.read_nwb_recording(
66
+ s3_url,
67
+ stream_mode="fsspec",
68
+ electrical_series_path="acquisition/ElectricalSeriesProbe00AP",
69
+ )
70
+ print("Original recording:")
71
+ print(rec0)
72
+
73
+ # slice 10 minutes near the end
74
+ #TODO better IBL sampling rate handling
75
+ rec = rec0
76
+ end_time = rec.get_end_time()
77
+ start_frame = rec.time_to_sample_index(end_time - 30 * 60)
78
+ end_frame = start_frame + 10 * 60 * 30_000
79
+ rec = rec.frame_slice(start_frame, end_frame)
80
+ rec.reset_times()
81
+ rec._sampling_frequency = 30_000
82
+
83
+ rec = dartsort.preprocess(rec, strategy="ibllike")
84
+ rec = rec.save_to_folder(preprocessing_path)
85
+
86
+ # %%
87
+ plt.imshow(
88
+ rec.get_traces(0, 10_000, 11_000).T,
89
+ aspect='auto',
90
+ vmin=-5,
91
+ vmax=5,
92
+ interpolation='nearest',
93
+ )
94
+ plt.colorbar(shrink=0.5, label='standardized voltage')
95
+ plt.xlabel('time (samples')
96
+ plt.ylabel('channels');
97
+
98
+ # %% [markdown]
99
+ # ## Run *dartsort*
100
+
101
+ # %%
102
+ ds_dir = experiment_path / "dartsort"
103
+ ds_res = dartsort.dartsort(rec, ds_dir)
104
+
105
+ # %% [markdown]
106
+ # # Visualize
107
+
108
+ # %%
109
+ dartvis.scatter_spike_features(sorting=ds_res["sorting"]);
110
+
111
+ # %%
@@ -72,8 +72,10 @@ include = 'src\/.*\.pyi?$|tests\/.*\.pyi?$'
72
72
 
73
73
  [tool.ruff]
74
74
  line-length = 88
75
- extend-select = ["I"]
76
- ignore = ["F722"]
75
+
76
+ [tool.ruff.lint]
77
+ extend-select = ["E", "F", "I", "N", "B", "SIM", "NPY", "RUF", "PTH", "PD", "FURB"]
78
+ ignore = ["F722", "N812", "SIM102", "SIM108", "E501", "N806", "N803", "N815"]
77
79
 
78
80
  [tool.jupytext]
79
81
  "formats" = "ipynb,py:percent"
@@ -47,8 +47,8 @@ logger = get_logger(__name__)
47
47
  class Agglomeration:
48
48
  agglomerated_sorting: DARTsortSorting
49
49
  merge_mapping: np.ndarray
50
- distances: np.ndarray
51
- shifts: np.ndarray
50
+ distances: np.ndarray | None
51
+ shifts: np.ndarray | None
52
52
  firing_corr: np.ndarray | None
53
53
 
54
54
 
@@ -71,38 +71,69 @@ def agglomerate(
71
71
  template_merge_cfg = refinement_cfg.template_merge_cfg
72
72
 
73
73
  if template_data is None:
74
- sorting = sorting.flatten()
75
-
76
- tdist = template_distances(
77
- sorting=sorting,
78
- recording=recording,
79
- motion=motion,
80
- template_data=template_data,
81
- waveform_cfg=waveform_cfg,
82
- template_merge_cfg=template_merge_cfg,
83
- computation_cfg=computation_cfg,
84
- )
74
+ did_flatten = True
75
+ sorting = sorting.flatten(include_gmm_properties=True)
76
+ else:
77
+ did_flatten = False
85
78
 
86
- # if not doing any QDA, be done now.
87
- if refinement_cfg is None or not refinement_cfg.qda_threshold:
88
- agg_sorting, new_ids = recluster(
79
+ if template_merge_cfg is not None:
80
+ tdist = template_distances(
89
81
  sorting=sorting,
90
- unit_ids=tdist.template_data.unit_ids,
91
- dists=tdist.distances,
92
- shifts=tdist.shifts,
93
- unit_snrs=tdist.template_data.snrs_by_channel().max(1),
94
- threshold=template_merge_cfg.merge_distance_threshold,
95
- link=template_merge_cfg.linkage,
82
+ recording=recording,
83
+ motion=motion,
84
+ template_data=template_data,
85
+ waveform_cfg=waveform_cfg,
86
+ template_merge_cfg=template_merge_cfg,
87
+ computation_cfg=computation_cfg,
96
88
  )
89
+ else:
90
+ tdist = None
91
+
92
+ # if not doing any QDA, be done now.
93
+ no_further_glom = (
94
+ (refinement_cfg is None)
95
+ or (template_merge_cfg is None)
96
+ or (not refinement_cfg.qda_threshold)
97
+ )
98
+ if no_further_glom:
99
+ if tdist is not None:
100
+ assert template_merge_cfg is not None
101
+ agg_sorting, new_ids = recluster(
102
+ sorting=sorting,
103
+ unit_ids=tdist.template_data.unit_ids,
104
+ dists=tdist.distances,
105
+ shifts=tdist.shifts,
106
+ unit_snrs=tdist.template_data.snrs_by_channel().max(1),
107
+ threshold=template_merge_cfg.merge_distance_threshold,
108
+ link=template_merge_cfg.linkage,
109
+ )
110
+ elif not did_flatten:
111
+ agg_sorting = sorting.flatten(include_gmm_properties=True)
112
+ new_ids = None
113
+ else:
114
+ agg_sorting = sorting
115
+ new_ids = None
116
+
117
+ if refinement_cfg is not None:
118
+ agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
119
+
120
+ agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
121
+ if new_ids is None:
122
+ new_ids = reorder
123
+ else:
124
+ new_ids = reorder[np.unique(new_ids, return_inverse=True)[1]]
125
+
97
126
  return Agglomeration(
98
127
  agglomerated_sorting=agg_sorting,
99
128
  merge_mapping=new_ids,
100
- distances=tdist.distances,
101
- shifts=tdist.shifts,
129
+ distances=None if tdist is None else tdist.distances,
130
+ shifts=None if tdist is None else tdist.shifts,
102
131
  firing_corr=None,
103
132
  )
104
133
 
105
134
  # tdist tells us the possible merges
135
+ assert tdist is not None
136
+ assert template_merge_cfg is not None
106
137
  distance_mask = linkage_mask(
107
138
  tdist.distances,
108
139
  linkage_method=template_merge_cfg.linkage,
@@ -119,7 +150,7 @@ def agglomerate(
119
150
  _oldsum = distance_mask[np.triu_indices_from(distance_mask)].sum()
120
151
  fcorr_mask = fcorr <= refinement_cfg.glom_max_firing_corr
121
152
  mask = np.logical_and(distance_mask, fcorr_mask)
122
- np.fill_diagonal(mask, True)
153
+ np.fill_diagonal(mask, val=True)
123
154
  _newsum = mask[np.triu_indices_from(mask)].sum()
124
155
  logger.dartsortdebug(
125
156
  f"Firing corr dropped QDA candidate count from {_oldsum} -> {_newsum}."
@@ -193,7 +224,7 @@ def agglomerate(
193
224
  final_mask = np.logical_or(qda_mask, force_mask)
194
225
  if si_mask is not None:
195
226
  final_mask = np.logical_or(final_mask, si_mask)
196
- np.fill_diagonal(final_mask, True)
227
+ np.fill_diagonal(final_mask, val=True)
197
228
  final_mask_as_distance = np.logical_not(final_mask).astype(np.float32)
198
229
 
199
230
  agg_sorting, new_ids = recluster(
@@ -615,7 +646,7 @@ def _qda_job(ij):
615
646
  try:
616
647
  kde = FFTKDE(bw="ISJ").fit(dll)
617
648
  except ValueError as e:
618
- logger.dartsortdebug(f"KDEpy error: {str(e)}")
649
+ logger.dartsortdebug(f"KDEpy error: {e}")
619
650
  p.score[i, j] = p.score[j, i] = 0.0
620
651
  p.min_ratio[i, j] = p.min_ratio[j, i] = 0.0
621
652
  return
@@ -722,7 +753,7 @@ def count_radial_weights(sorting: DARTsortSorting, motion: MotionInfo, radius: f
722
753
  vv = row[ii]
723
754
  vv = vv / vv.sum()
724
755
 
725
- for channel, value in zip(ii, vv):
756
+ for channel, value in zip(ii, vv, strict=True):
726
757
  cixs = ci[channel]
727
758
  cixs = cixs[cixs < motion.rgeom.shape[0]]
728
759
  weights[uu, cixs] += value
@@ -734,7 +765,7 @@ def count_radial_weights(sorting: DARTsortSorting, motion: MotionInfo, radius: f
734
765
 
735
766
  def firing_corr(sorting: DARTsortSorting, dt: float, method="binsqrt"):
736
767
  if method != "binsqrt":
737
- assert False
768
+ raise ValueError(f"Unknown {method=}.")
738
769
 
739
770
  tsg = sorting.to_tsgroup()
740
771
  fr = tsg.count(bin_size=dt) / dt
@@ -751,6 +782,8 @@ def combine_gmm_scores(
751
782
  responsibilities = getattr(sorting, f"{old_prefix}_responsibilities", None)
752
783
  logliks = getattr(sorting, f"{old_prefix}_log_liks", None)
753
784
 
785
+ assert not hasattr(sorting, f"{new_prefix}_responsibilities")
786
+
754
787
  havec = candidates is not None
755
788
  haver = responsibilities is not None
756
789
  havel = logliks is not None
@@ -780,10 +813,9 @@ def combine_gmm_scores(
780
813
  assert _maxdiff <= 1e-3, _maxdiff
781
814
  assert np.greater_equal(np.isneginf(logliks[:, :-1]), candidates == -1).all()
782
815
  if sorting.labels is not None:
783
- assert np.all(
784
- np.logical_or(
785
- sorting.labels < 0, sorting.labels == new_ids[candidates[:, 0]]
786
- )
816
+ not_noise = np.flatnonzero(sorting.labels >= 0)
817
+ assert np.array_equal(
818
+ sorting.labels[not_noise], new_ids[candidates[not_noise, 0]]
787
819
  )
788
820
 
789
821
  # two steps: first merge, then sort
@@ -839,20 +871,24 @@ def _combine_loop(
839
871
  mergedl: np.ndarray,
840
872
  ):
841
873
  for s in numba.prange(cand.shape[0]): # ty: ignore
842
- rcand = cand[s]
874
+ spike_cand = cand[s]
843
875
  for j in range(cand.shape[1] - 1):
844
- ncandj = rcand[j]
845
- if ncandj < 0 or new_id_counts[ncandj] <= 1:
876
+ spike_candj = spike_cand[j]
877
+
878
+ # noise or not a merge
879
+ if spike_candj < 0 or new_id_counts[spike_candj] <= 1:
846
880
  continue
847
881
 
848
- eq_ncandj = rcand[j + 1 :] == ncandj
849
- if eq_ncandj.sum() < 1:
882
+ # what later indices are equal to me?
883
+ eq_spike_candj = spike_cand[j + 1 :] == spike_candj
884
+ if eq_spike_candj.sum() < 1:
850
885
  continue
851
886
 
887
+ # loop through and combine liks/resps, and -1 out the cands
852
888
  rsum = mergedr[s, j]
853
889
  lsum = mergedl[s, j]
854
890
  for i, k in enumerate(range(j + 1, cand.shape[1])):
855
- if not eq_ncandj[i]:
891
+ if not eq_spike_candj[i]:
856
892
  continue
857
893
  cand[s, k] = -1
858
894
 
@@ -867,6 +903,18 @@ def _combine_loop(
867
903
  mergedr[s, j] = rsum
868
904
  mergedl[s, j] = lsum
869
905
 
906
+ # vacuum into noise component
907
+ # this is partly to handle stuff that was missed before getting here
908
+ # from flatten, for example
909
+ for j in range(cand.shape[1]):
910
+ if 0 <= spike_cand[j] < new_id_counts.shape[0]:
911
+ continue
912
+ rsj = mergedr[s, j]
913
+ if rsj == 0:
914
+ continue
915
+ mergedr[s, -1] += rsj
916
+ mergedr[s, j] = 0.0
917
+
870
918
 
871
919
  def deduplicate_spikes(
872
920
  sorting: DARTsortSorting,
@@ -1028,7 +1076,7 @@ def coentropy_merge_mask(
1028
1076
 
1029
1077
  mask = np.logical_or(c.cov >= coverage_threshold, c.iou >= iou_threshold)
1030
1078
  mask = np.logical_and(c.coentropy >= min_coentropy, mask)
1031
- np.fill_diagonal(mask, True)
1079
+ np.fill_diagonal(mask, val=True)
1032
1080
  return mask, c
1033
1081
 
1034
1082
 
@@ -74,7 +74,7 @@ def apply_reclustering(
74
74
 
75
75
  # align to best snr unit
76
76
  times_updated = sorting.times_samples.copy()
77
- for new_label, orig_labels in clust_inverse.items():
77
+ for orig_labels in clust_inverse.values():
78
78
  # we don't need to realign clusters which didn't change
79
79
  if len(orig_labels) <= 1:
80
80
  continue
@@ -170,7 +170,9 @@ def linkage_mask(
170
170
  return mask
171
171
 
172
172
 
173
- def sparsify_labels(labels: np.ndarray, ids: np.ndarray | None = None) -> dict[int, np.ndarray]:
173
+ def sparsify_labels(
174
+ labels: np.ndarray, ids: np.ndarray | None = None
175
+ ) -> dict[int, np.ndarray]:
174
176
  assert labels.ndim == 1
175
177
  if ids is None:
176
178
  ids = np.unique(labels)
@@ -186,7 +188,7 @@ def leafsets(Z, max_distance=np.inf):
186
188
  n = len(Z) + 1
187
189
  leaves = {}
188
190
  for i, row in enumerate(Z):
189
- pa, pb, dist, nab = row
191
+ pa, pb, dist, _nab = row
190
192
  if dist > max_distance:
191
193
  break
192
194
  leavesa = leaves.get(pa, [int(pa)])
@@ -206,7 +208,7 @@ def maximal_leaf_groups(
206
208
  leaves = {k: set(v) for k, v in leaves.items()}
207
209
  group_parents = []
208
210
  for i, row in reversed(list(enumerate(Z))):
209
- pa, pb, dist, nab = row
211
+ _pa, _pb, _dist, nab = row
210
212
  if nab > max_group_size:
211
213
  continue
212
214
  if n + i not in leaves:
@@ -278,7 +280,7 @@ def is_largest_set_smaller_than(Z, leaf_descendants, max_size=5):
278
280
  # at each branch, if #descendents<size, branch gets true and
279
281
  # its parents are set to false. that way only one ancestor
280
282
  # is true for each leaf.
281
- for i, (pa, pb, dist, nab) in enumerate(Z):
283
+ for i, (pa, pb, _dist, _nab) in enumerate(Z):
282
284
  sz = len(leaf_descendants[n_units + i])
283
285
  if sz > max_size:
284
286
  continue
@@ -309,7 +311,7 @@ def combine_distances(
309
311
  has threshold 1.
310
312
  """
311
313
  dists = distances[0] / thresholds[0]
312
- for dist, thresh in zip(distances[1:], thresholds[1:]):
314
+ for dist, thresh in zip(distances[1:], thresholds[1:], strict=True):
313
315
  dists = agg_function(dists, dist / thresh)
314
316
  return sym_function(dists, dists.T)
315
317
 
@@ -525,7 +527,7 @@ def get_main_channel_pcs(
525
527
  return features
526
528
 
527
529
 
528
- def decrumb(labels: np.ndarray, min_size: int=5, in_place=False, flatten=True):
530
+ def decrumb(labels: np.ndarray, min_size: int = 5, in_place=False, flatten=True):
529
531
  """Remove small units
530
532
 
531
533
  Parameters
@@ -85,7 +85,7 @@ def get_smoothed_density(
85
85
  # figure out what bandwidths we'll be working on
86
86
 
87
87
  if do_ramp := bool(sigma_low):
88
- min_sigma = min(sigma, sigma_low) # type: ignore
88
+ min_sigma = min(sigma, sigma_low) # type: ignore # ty: ignore[x]
89
89
  else:
90
90
  min_sigma = sigma
91
91
 
@@ -3,7 +3,7 @@ import torch
3
3
  import torch.nn.functional as F
4
4
  from torch import Tensor
5
5
 
6
- from dartsort.util.py_util import databag
6
+ from ..util.py_util import databag, panic
7
7
 
8
8
  try:
9
9
  import cupy # type: ignore # ty: ignore[x]
@@ -58,7 +58,7 @@ def kmeanspp(
58
58
  closest = torch.cdist(X, X.mean(0, keepdim=True)).argmax()
59
59
  centroid_ixs[0] = closest.item()
60
60
  else:
61
- assert False
61
+ panic()
62
62
 
63
63
  if Xnormsq is None:
64
64
  Xnormsq = torch.linalg.vector_norm(X, dim=1).square_()
@@ -227,7 +227,7 @@ def truncated_kmeans(
227
227
  log_likelihoods = None
228
228
 
229
229
  if show_progress:
230
- it = progrange(n_iter, desc=f"kmeans σ={sigma:0.4f}")
230
+ it = progrange(n_iter, desc=f"kmeans sigma={sigma:0.4f}")
231
231
  else:
232
232
  it = range(n_iter)
233
233
 
@@ -323,7 +323,7 @@ def truncated_kmeans(
323
323
  break
324
324
  prev_sigma = sigma
325
325
  if show_progress:
326
- it.set_description(f"kmeans σ={sigma:0.4f}") # type: ignore
326
+ it.set_description(f"kmeans sigma={sigma:0.4f}") # type: ignore
327
327
 
328
328
  return dict(
329
329
  centroid_ixs=centroid_ixs,
@@ -447,7 +447,7 @@ def kmeans(
447
447
  if weights is not None:
448
448
  weights = weights.to(device=X.device)
449
449
  with torch.jit.optimized_execution(True):
450
- for j in range(n_kmeans_tries):
450
+ for _ in range(n_kmeans_tries):
451
451
  aa, ee, cc, dists = kmeans_inner(
452
452
  X,
453
453
  n_kmeanspp_tries=n_kmeanspp_tries,
@@ -810,7 +810,7 @@ def truncated_kmeans_from_labels(
810
810
  distsq_buf = (distsq_buf, torch.zeros_like(distsq_buf))
811
811
 
812
812
  if show_progress:
813
- it = progrange(n_iter, desc=f"kmeans σ={sigma:0.4f}")
813
+ it = progrange(n_iter, desc=f"kmeans sigma={sigma:0.4f}")
814
814
  else:
815
815
  it = range(n_iter)
816
816
 
@@ -891,7 +891,7 @@ def truncated_kmeans_from_labels(
891
891
 
892
892
  prev_sigma = sigma
893
893
  if show_progress:
894
- it.set_description(f"kmeans σ={sigma:0.4f}") # type: ignore
894
+ it.set_description(f"kmeans sigma={sigma:0.4f}") # type: ignore
895
895
 
896
896
  return KMeansResult(
897
897
  labels=labels,