DARTsort 0.5.26__tar.gz → 0.5.28__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. {dartsort-0.5.26 → dartsort-0.5.28}/PKG-INFO +1 -1
  2. {dartsort-0.5.26 → dartsort-0.5.28}/pyproject.toml +1 -0
  3. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/clustering/density.py +9 -5
  4. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/clustering/mixture.py +2 -2
  5. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/config.py +7 -2
  6. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/evaluate/hybrid_util.py +1 -1
  7. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/evaluate/hybridkit.py +2 -1
  8. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/evaluate/noise_recording_tools.py +1 -1
  9. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/evaluate/simkit.py +1 -1
  10. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/evaluate/simlib.py +5 -5
  11. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/main.py +11 -2
  12. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/peel_lib.py +58 -27
  13. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/shaver.py +2 -2
  14. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/subtract.py +42 -45
  15. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/subtract_util.py +127 -52
  16. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/threshold.py +146 -18
  17. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/templates/realignment.py +4 -3
  18. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/templates/templib.py +2 -1
  19. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/decollider.py +130 -66
  20. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/matched_filter_net.py +1 -2
  21. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/pipeline.py +19 -1
  22. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/temporal_pca.py +23 -17
  23. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/transform_base.py +5 -25
  24. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/data_util.py +3 -3
  25. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/internal_config.py +37 -11
  26. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/noise_util.py +5 -5
  27. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/preprocess_util.py +36 -3
  28. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/registration_util.py +1 -1
  29. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/__init__.py +2 -2
  30. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/analysis_plots.py +14 -6
  31. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/gt.py +1 -1
  32. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/vismain.py +2 -0
  33. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort.egg-info/PKG-INFO +1 -1
  34. {dartsort-0.5.26 → dartsort-0.5.28}/tests/conftest.py +3 -3
  35. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_config.py +1 -1
  36. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_hybridkit.py +1 -1
  37. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_matching.py +4 -3
  38. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_modeling.py +7 -0
  39. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_subtract.py +106 -15
  40. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_templates.py +2 -2
  41. dartsort-0.5.28/tests/test_threshold.py +80 -0
  42. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_transform.py +13 -2
  43. dartsort-0.5.26/tests/test_threshold.py +0 -9
  44. {dartsort-0.5.26 → dartsort-0.5.28}/.github/workflows/ci.yml +0 -0
  45. {dartsort-0.5.26 → dartsort-0.5.28}/.github/workflows/ci_windows.yml +0 -0
  46. {dartsort-0.5.26 → dartsort-0.5.28}/.github/workflows/deploy.yml +0 -0
  47. {dartsort-0.5.26 → dartsort-0.5.28}/.github/workflows/docs.yml +0 -0
  48. {dartsort-0.5.26 → dartsort-0.5.28}/.gitignore +0 -0
  49. {dartsort-0.5.26 → dartsort-0.5.28}/README.md +0 -0
  50. {dartsort-0.5.26 → dartsort-0.5.28}/docs/index.md +0 -0
  51. {dartsort-0.5.26 → dartsort-0.5.28}/docs/main_api.md +0 -0
  52. {dartsort-0.5.26 → dartsort-0.5.28}/docs/toolkit_api.md +0 -0
  53. {dartsort-0.5.26 → dartsort-0.5.28}/environment.yml +0 -0
  54. {dartsort-0.5.26 → dartsort-0.5.28}/notebook/.keep +0 -0
  55. {dartsort-0.5.26 → dartsort-0.5.28}/notebook/fit_and_batch_matching.py +0 -0
  56. {dartsort-0.5.26 → dartsort-0.5.28}/notebook/ibl_dandi_example.py +0 -0
  57. {dartsort-0.5.26 → dartsort-0.5.28}/setup.cfg +0 -0
  58. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/__init__.py +0 -0
  59. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/__main__.py +0 -0
  60. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/cli.py +0 -0
  61. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/clustering/__init__.py +0 -0
  62. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/clustering/agglomerate.py +0 -0
  63. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/clustering/cluster_util.py +0 -0
  64. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/clustering/clustering.py +0 -0
  65. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/clustering/clustering_features.py +0 -0
  66. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/clustering/forward_backward.py +0 -0
  67. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/clustering/kmeans.py +0 -0
  68. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/clustering/merge.py +0 -0
  69. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/clustering/refine_util.py +0 -0
  70. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/detect/__init__.py +0 -0
  71. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/detect/detect.py +0 -0
  72. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/evaluate/__init__.py +0 -0
  73. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/evaluate/analysis.py +0 -0
  74. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/evaluate/comparison.py +0 -0
  75. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  76. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/localize/__init__.py +0 -0
  77. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/localize/localize_torch.py +0 -0
  78. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/localize/localize_util.py +0 -0
  79. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/__init__.py +0 -0
  80. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/grab.py +0 -0
  81. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/matching.py +0 -0
  82. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/matching_util/__init__.py +0 -0
  83. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  84. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/matching_util/drifty.py +0 -0
  85. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  86. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  87. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  88. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/peel_base.py +0 -0
  89. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/peel/reduction_template.py +0 -0
  90. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  91. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/pretrained/detect_np1.pt +0 -0
  92. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/pretrained/detect_np2.pt +0 -0
  93. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  94. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  95. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  96. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  97. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  98. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  99. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/templates/__init__.py +0 -0
  100. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/templates/get_templates.py +0 -0
  101. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/templates/postprocess_util.py +0 -0
  102. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/templates/template_util.py +0 -0
  103. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/templates/templates.py +0 -0
  104. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/__init__.py +0 -0
  105. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  106. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/all_transformers.py +0 -0
  107. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/amortized_localization.py +0 -0
  108. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/amplitudes.py +0 -0
  109. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/denoising_scorer.py +0 -0
  110. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/enforce_decrease.py +0 -0
  111. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/fixed_prop.py +0 -0
  112. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/interp.py +0 -0
  113. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/localize.py +0 -0
  114. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/matching_denoiser.py +0 -0
  115. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/mixture_classifier.py +0 -0
  116. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/reduction.py +0 -0
  117. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  118. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/supervised_denoiser.py +0 -0
  119. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/vq_matched_filter.py +0 -0
  120. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/transform/whiten.py +0 -0
  121. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/__init__.py +0 -0
  122. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/cli_util.py +0 -0
  123. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/drift_util.py +0 -0
  124. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/interpolation_util.py +0 -0
  125. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/job_util.py +0 -0
  126. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/logging_util.py +0 -0
  127. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/main_util.py +0 -0
  128. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/more_operators.py +0 -0
  129. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/motion.py +0 -0
  130. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/multiprocessing_util.py +0 -0
  131. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/nn_util.py +0 -0
  132. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/peel_util.py +0 -0
  133. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/py_util.py +0 -0
  134. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/residual_util.py +0 -0
  135. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/sparse_util.py +0 -0
  136. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/spikeio.py +0 -0
  137. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/spiketorch.py +0 -0
  138. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/testing_util/__init__.py +0 -0
  139. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  140. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  141. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/torch_optimization_util.py +0 -0
  142. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/torch_util.py +0 -0
  143. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/util/waveform_util.py +0 -0
  144. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/colors.py +0 -0
  145. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/layout.py +0 -0
  146. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/metrics_util.py +0 -0
  147. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/mixture.py +0 -0
  148. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/recanim.py +0 -0
  149. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/scatterplots.py +0 -0
  150. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/sorting.py +0 -0
  151. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/unit.py +0 -0
  152. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/unit_comparison.py +0 -0
  153. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/versus.py +0 -0
  154. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort/vis/waveforms.py +0 -0
  155. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort.egg-info/SOURCES.txt +0 -0
  156. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort.egg-info/dependency_links.txt +0 -0
  157. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort.egg-info/entry_points.txt +0 -0
  158. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort.egg-info/requires.txt +0 -0
  159. {dartsort-0.5.26 → dartsort-0.5.28}/src/dartsort.egg-info/top_level.txt +0 -0
  160. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_agglomerate.py +0 -0
  161. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_alignment.py +0 -0
  162. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_cluster_util.py +0 -0
  163. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_clustering.py +0 -0
  164. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_clustering_features.py +0 -0
  165. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_dartsort.py +0 -0
  166. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_data_util.py +0 -0
  167. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_density.py +0 -0
  168. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_detect.py +0 -0
  169. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_drift_util.py +0 -0
  170. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_drifty_matching.py +0 -0
  171. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_enforce_decrease.py +0 -0
  172. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_grab_and_featurize.py +0 -0
  173. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_kmeans.py +0 -0
  174. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_linalg.py +0 -0
  175. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_localize_torch.py +0 -0
  176. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_mixture_classifier.py +0 -0
  177. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_simkit.py +0 -0
  178. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_spiketorch.py +0 -0
  179. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_torch_optimization_util.py +0 -0
  180. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_util.py +0 -0
  181. {dartsort-0.5.26 → dartsort-0.5.28}/tests/test_waveform_util.py +0 -0
  182. {dartsort-0.5.26 → dartsort-0.5.28}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.26
3
+ Version: 0.5.28
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -97,3 +97,4 @@ replace-imports-with-any = ["hdbscan"]
97
97
  [tool.ty.rules]
98
98
  "unsupported-base" = "ignore"
99
99
  "possibly-unresolved-reference" = "error"
100
+ "invalid-key" = "ignore"
@@ -64,7 +64,7 @@ def kdtree_inliers(
64
64
 
65
65
  def get_smoothed_density(
66
66
  X,
67
- inliers: np.ndarray | slice = slice(None),
67
+ inliers: np.ndarray | slice = slice(None), # noqa: B008
68
68
  sigma: float = 1.0,
69
69
  sigma_low: float | None = None,
70
70
  sigma_ramp_ax=-1,
@@ -154,7 +154,7 @@ def get_smoothed_density(
154
154
  def get_smoothed_density_ratio(
155
155
  X,
156
156
  *,
157
- inliers: np.ndarray | slice = slice(None),
157
+ inliers: np.ndarray | slice = slice(None), # noqa: B008
158
158
  sigmas: list[float],
159
159
  sigma_lows: list[float] | list[None] | None = None,
160
160
  sigma_ramp_ax=-1,
@@ -504,7 +504,7 @@ def density_peaks(
504
504
  remove_borders=False,
505
505
  border_search_radius=10.0,
506
506
  border_search_neighbors=3,
507
- inlier_dims=slice(0, 2),
507
+ inlier_dims=slice(0, 2), # noqa: B008
508
508
  leafsize=24,
509
509
  density_strategy="sort",
510
510
  workers=-1,
@@ -599,7 +599,9 @@ def density_peaks(
599
599
  )
600
600
 
601
601
  if remove_clusters_smaller_than:
602
- labels = decrumb_labels(labels, min_size=remove_clusters_smaller_than, in_place=True)
602
+ labels = decrumb_labels(
603
+ labels, min_size=remove_clusters_smaller_than, in_place=True
604
+ )
603
605
 
604
606
  return dict(density=density, nhdn=nhdn, labels=labels, kdtree=kdtree)
605
607
 
@@ -796,7 +798,9 @@ def gmm_density_peaks(
796
798
  if remove_clusters_smaller_than:
797
799
  if show_progress:
798
800
  logger.info("Clean...")
799
- labels = decrumb_labels(labels, min_size=remove_clusters_smaller_than, in_place=True)
801
+ labels = decrumb_labels(
802
+ labels, min_size=remove_clusters_smaller_than, in_place=True
803
+ )
800
804
  res["labels"] = labels
801
805
 
802
806
  if mop:
@@ -7598,7 +7598,7 @@ def _update_lut_mean_batch(
7598
7598
 
7599
7599
  # constplogdet. add in the signal-rank-0-only terms.
7600
7600
  lut_params.constplogdet[i0:i1] = neighb_cov.nobs[nn].mul_(LOG_2PI) # type: ignore # ty: ignore[x]
7601
- lut_params.constplogdet[i0:i1] += neighb_cov.b.logdet[nn] # ty: ignore[not-subscriptable]
7601
+ lut_params.constplogdet[i0:i1] += neighb_cov.b.logdet[nn]
7602
7602
  if pnoid:
7603
7603
  assert lut_params.constplogdet[i0:i1].isfinite().all() # type: ignore # ty: ignore[x]
7604
7604
 
@@ -7658,7 +7658,7 @@ def _update_lut_ppca_batch(
7658
7658
  assert lut_params.Tpad is not None
7659
7659
  lut_params.Tpad[i0:i1, :, :-1] = T # type: ignore # ty: ignore[x]
7660
7660
  cap_logdet = L.diagonal(dim1=-2, dim2=-1).log().sum(dim=1).mul_(2.0)
7661
- lut_params.constplogdet[i0:i1] += cap_logdet # ty: ignore[not-subscriptable]
7661
+ lut_params.constplogdet[i0:i1] += cap_logdet
7662
7662
  if pnoid:
7663
7663
  assert lut_params.b.constplogdet[i0:i1].isfinite().all()
7664
7664
 
@@ -4,6 +4,7 @@ from typing import Annotated, Literal
4
4
  from pydantic import Field
5
5
 
6
6
  from .util.internal_config import (
7
+ DetectionProposal,
7
8
  InterpKernel,
8
9
  InterpMethod,
9
10
  KmeansppSelection,
@@ -12,6 +13,7 @@ from .util.internal_config import (
12
13
  PreprocessingStrategy,
13
14
  RealignStrategy,
14
15
  TemplateSVDMethod,
16
+ TPCADenoiserFitStage,
15
17
  WhiteningEstimator,
16
18
  WhiteningStrategy,
17
19
  default_pretrained_path, # noqa
@@ -277,10 +279,13 @@ class DeveloperConfig(DARTsortUserConfig):
277
279
  nn_denoiser_noise_waveforms: int = 100 * 256
278
280
  nn_denoiser_extra_kwargs: dict | None = None
279
281
  score_filter_radius_um: float = 0.0
280
- detection_proposal: Literal["voltage", "tpca", "vq", "score_net"] = "voltage"
282
+ detection_proposal: DetectionProposal = "voltage"
281
283
  proposal_filters: int = 1
282
- score_proposal_threshold: float = 5.0
284
+ propose_with_score_net: bool = False
285
+ proposal_threshold: float = 5.0
286
+ score_net_threshold: float = 5.0
283
287
  do_tpca_denoise: bool = True
288
+ tpca_denoise_fit: TPCADenoiserFitStage | None = None
284
289
  first_denoiser_thinning: float = 0.0
285
290
  first_denoiser_spatial_dedup_radius: float = 100.0
286
291
  realign_to_denoiser: bool = True
@@ -118,7 +118,7 @@ def precompute_displaced_registered_templates(
118
118
  geometry: np.ndarray,
119
119
  displacements: np.ndarray,
120
120
  sampling_frequency: float = 30000,
121
- template_subset=slice(None),
121
+ template_subset=slice(None), # noqa: B008
122
122
  ) -> DriftingTemplates:
123
123
  """Use spikeinterface tools to turn templates on registered geom into
124
124
  precomputed drifting templates on the regular geom.
@@ -134,7 +134,8 @@ def make_hybrid_recording(
134
134
  assert injected_sorting._recording is None
135
135
  assert target_recording.get_num_segments() == 1
136
136
  assert injected_sorting.get_num_segments() == 1
137
- assert injected_sorting.get_last_spike_frame() < target_recording.get_num_frames()
137
+ if hasattr(injected_sorting, "get_last_spike_frame"):
138
+ assert injected_sorting.get_last_spike_frame() < target_recording.get_num_frames() # ty:ignore[call-non-callable]
138
139
  assert abs(templates.sampling_frequency - target_sampling_frequency) < 10
139
140
 
140
141
  if folder is not None and not overwrite:
@@ -71,7 +71,7 @@ def get_background_recording(
71
71
  spatial_std, spatial_vt = rbf_kernel_sqrt(
72
72
  geom, bandwidth=noise_spatial_kernel_bandwidth
73
73
  )
74
- noise_temporal_kernel = noise_temporal_kernel.astype(spatial_vt.dtype) + 0j
74
+ noise_temporal_kernel = noise_temporal_kernel.astype(spatial_vt.dtype) + 0j # ty:ignore[no-matching-overload]
75
75
  noise = StationaryFactorizedNoise(
76
76
  spatial_std=spatial_std,
77
77
  vt_spatial=spatial_vt,
@@ -90,7 +90,7 @@ def generate_simulation(
90
90
  if folder is not None and not (overwrite or just_noise or no_save):
91
91
  try:
92
92
  return load_simulation(folder)
93
- except Exception: # noqa: BLE001, S110
93
+ except Exception:
94
94
  pass
95
95
 
96
96
  if noise_recording_folder is not None:
@@ -3,7 +3,7 @@ import sys
3
3
  import warnings
4
4
  from dataclasses import replace
5
5
  from pathlib import Path
6
- from typing import TYPE_CHECKING, Literal, cast
6
+ from typing import TYPE_CHECKING, Any, Literal, cast
7
7
 
8
8
  import h5py
9
9
  import numpy as np
@@ -343,7 +343,7 @@ def add_features(h5_path, recording, featurization_cfg, computation_cfg):
343
343
  _, feats = gt_pipeline(
344
344
  chunk, channels=cast(h5py.Dataset, h5["channels"])[sli]
345
345
  )
346
- for k in f_dsets: # noqa: PLC0206
346
+ for k in f_dsets:
347
347
  f_dsets[k][sli] = feats[k].numpy(force=True)
348
348
 
349
349
 
@@ -834,11 +834,11 @@ class InjectSpikesPreprocessor(BasePreprocessor):
834
834
 
835
835
  # arrays discovered in batches below
836
836
  f_dt = self.features_dtype
837
- inj_wf_shape = (
837
+ inj_wf_shape: tuple[int, int] = (
838
838
  self.spike_length_samples,
839
839
  self.extract_channel_index.shape[1],
840
840
  )
841
- dataset_shapes = {
841
+ dataset_shapes: dict[str, tuple[tuple[int, ...], Any]] = {
842
842
  "localizations": ((3,), f_dt),
843
843
  "displacements": ((), f_dt),
844
844
  "ptp_amplitudes": ((), f_dt),
@@ -1004,7 +1004,7 @@ class InjectSpikesPreprocessor(BasePreprocessor):
1004
1004
  try:
1005
1005
  recording = read_binary_folder(recording_dir)
1006
1006
  logger.info("Loaded %s", recording_dir)
1007
- except Exception: # noqa: BLE001
1007
+ except Exception:
1008
1008
  recording = None
1009
1009
  else:
1010
1010
  recording = None
@@ -140,10 +140,19 @@ def dartsort(
140
140
  ds_handle_link_from(cfg, output_dir)
141
141
 
142
142
  # preprocess
143
- copy_rec_flag = ds_will_copy_recording(cfg, recording)
143
+ # saves provenance info (lightweight) about the preprocessing, including
144
+ # noise levels in standardization etc to a json file
144
145
  recording = preprocess(
145
- recording, cfg.preprocessing, cfg.already_preprocessed, cfg.preprocessing_dtype
146
+ recording,
147
+ output_dir=output_dir,
148
+ strategy=cfg.preprocessing,
149
+ already_preprocessed=cfg.already_preprocessed,
150
+ dtype=cfg.preprocessing_dtype,
146
151
  )
152
+
153
+ # determine if recording will be saved to a tmpdir and warn the user
154
+ # if they're going to hit a slow case
155
+ copy_rec_flag = ds_will_copy_recording(cfg, recording)
147
156
  check_recording(recording, copy_flag=copy_rec_flag)
148
157
 
149
158
  needs_dir = copy_rec_flag or cfg.work_in_tmpdir
@@ -12,6 +12,7 @@ from torch import Tensor
12
12
  from ..detect import detect_and_deduplicate
13
13
  from ..util.internal_config import (
14
14
  ComputationConfig,
15
+ FeaturizationConfig,
15
16
  FitSamplingConfig,
16
17
  PeakSign,
17
18
  ThresholdingConfig,
@@ -20,7 +21,6 @@ from ..util.internal_config import (
20
21
  from ..util.job_util import ensure_computation_config
21
22
  from ..util.spiketorch import grab_spikes, subtract_spikes_
22
23
  from ..util.torch_util import torch_compile
23
- from ..util.waveform_util import make_channel_index
24
24
  from .peel_base import PeelingBatchResult
25
25
 
26
26
  if TYPE_CHECKING:
@@ -120,7 +120,7 @@ def flatten_denan_and_whiten_batched(
120
120
  def threshold_chunk(
121
121
  traces,
122
122
  channel_index,
123
- detection_threshold=4.0,
123
+ voltage_threshold=4.0,
124
124
  peak_sign: PeakSign = "both",
125
125
  peak_channel_index=None,
126
126
  dedup_channel_index=None,
@@ -140,10 +140,9 @@ def threshold_chunk(
140
140
  rg=None,
141
141
  quiet=False,
142
142
  ) -> PeelingBatchResult:
143
- n_index = channel_index.shape[1]
144
143
  times_rel, channels, energies = detect_and_deduplicate(
145
144
  traces,
146
- threshold=detection_threshold,
145
+ threshold=voltage_threshold,
147
146
  peak_channel_index=peak_channel_index,
148
147
  dedup_neighborhoods=dedup_channel_index,
149
148
  peak_sign=peak_sign,
@@ -153,6 +152,46 @@ def threshold_chunk(
153
152
  return_energies=True,
154
153
  trough_priority=trough_priority,
155
154
  )
155
+ return extract_detections(
156
+ traces,
157
+ times_rel,
158
+ channels,
159
+ energies,
160
+ channel_index=channel_index,
161
+ trough_offset_samples=trough_offset_samples,
162
+ spike_length_samples=spike_length_samples,
163
+ left_margin=left_margin,
164
+ right_margin=right_margin,
165
+ max_spikes_per_chunk=max_spikes_per_chunk,
166
+ thinning=thinning,
167
+ time_jitter=time_jitter,
168
+ spatial_jitter_channel_index=spatial_jitter_channel_index,
169
+ return_waveforms=return_waveforms,
170
+ rg=rg,
171
+ quiet=quiet,
172
+ )
173
+
174
+
175
+ def extract_detections(
176
+ traces,
177
+ times_rel,
178
+ channels,
179
+ energies,
180
+ *,
181
+ channel_index,
182
+ trough_offset_samples=42,
183
+ spike_length_samples=121,
184
+ left_margin=0,
185
+ right_margin=0,
186
+ max_spikes_per_chunk=None,
187
+ thinning=0.0,
188
+ time_jitter=0,
189
+ spatial_jitter_channel_index=None,
190
+ return_waveforms=True,
191
+ rg=None,
192
+ quiet=False,
193
+ ) -> PeelingBatchResult:
194
+ n_index = channel_index.shape[1]
156
195
  if not times_rel.numel():
157
196
  return PeelingBatchResult(
158
197
  n_spikes=0,
@@ -281,7 +320,7 @@ def perturb_detections(
281
320
  n = len(times_rel)
282
321
  if time_jitter:
283
322
  assert rg is not None
284
- jitter = rg.integers(low=-time_jitter, high=time_jitter + 1)
323
+ jitter = rg.integers(low=-time_jitter, high=time_jitter + 1, size=n)
285
324
  times_rel = times_rel + torch.asarray(
286
325
  jitter, dtype=times_rel.dtype, device=times_rel.device
287
326
  )
@@ -436,8 +475,8 @@ def threshold_to_fit(
436
475
  recording: BaseRecording,
437
476
  waveform_cfg: WaveformConfig,
438
477
  channel_index: Tensor,
439
- spatial_dedup_radius: float | None,
440
478
  threshold_cfg: ThresholdingConfig,
479
+ featurization_cfg: FeaturizationConfig,
441
480
  sampling_cfg: FitSamplingConfig,
442
481
  max_waveforms_fit: int | None = None,
443
482
  n_residual_snips: int | None = None,
@@ -448,34 +487,23 @@ def threshold_to_fit(
448
487
 
449
488
  Used by subtraction to fit initial NN denoisers.
450
489
  """
451
- from ..transform import Waveform, WaveformPipeline
452
490
  from ..util.data_util import subsample_waveforms
453
491
  from .threshold import Threshold
454
492
 
455
493
  computation_cfg = ensure_computation_config(computation_cfg)
456
494
 
457
- geom = recording.get_channel_locations()
458
- waveform_node = Waveform(
459
- channel_index=channel_index,
460
- waveform_cfg=waveform_cfg,
461
- sampling_frequency=recording.sampling_frequency,
462
- )
463
- waveform_pipeline = WaveformPipeline([waveform_node])
464
-
465
- if spatial_dedup_radius:
466
- dn_dedup_ci = make_channel_index(geom, spatial_dedup_radius, to_torch=True)
467
- dn_dedup_ci = dn_dedup_ci.to(channel_index)
468
- else:
469
- dn_dedup_ci = channel_index
470
- trainer = Threshold(
495
+ trainer = Threshold.from_config(
471
496
  recording=recording,
472
- channel_index=channel_index,
473
- featurization_pipeline=waveform_pipeline,
474
- p=threshold_cfg,
475
497
  waveform_cfg=waveform_cfg,
476
- dedup_channel_index=dn_dedup_ci,
477
- fit_sampling_cfg=sampling_cfg,
498
+ thresholding_cfg=threshold_cfg,
499
+ featurization_cfg=featurization_cfg,
500
+ sampling_cfg=sampling_cfg,
501
+ extract_channel_index=channel_index,
478
502
  )
503
+ assert trainer.featurization_pipeline is not None
504
+ assert "waveforms" in {
505
+ ds.name for ds in trainer.featurization_pipeline.spike_datasets()
506
+ }
479
507
 
480
508
  if max_waveforms_fit is None:
481
509
  max_waveforms_fit = sampling_cfg.max_waveforms_fit
@@ -490,6 +518,9 @@ def threshold_to_fit(
490
518
  with TemporaryDirectory(dir=tmp_dir) as temp_dir:
491
519
  temp_hdf5_filename = Path(temp_dir) / "subtraction_denoiser0_fit.h5"
492
520
  try:
521
+ trainer.load_or_fit_and_save_models(
522
+ Path(temp_dir) / "threshold_models", computation_cfg=computation_cfg
523
+ )
493
524
  trainer.run_subsampled_peeling(
494
525
  temp_hdf5_filename,
495
526
  stop_after_n_waveforms=max_waveforms_fit,
@@ -515,7 +546,7 @@ def threshold_to_fit(
515
546
  raise ValueError(
516
547
  "Found no spikes when thresholding to get model fitting data. "
517
548
  "This usually indicates a preprocessing issue, since it means "
518
- f"that no spikes could be found at threshold {threshold_cfg.detection_threshold}."
549
+ f"that no spikes could be found at threshold {threshold_cfg.voltage_threshold}."
519
550
  )
520
551
 
521
552
  # fit the thing
@@ -59,7 +59,7 @@ class Shaver(BasePeeler):
59
59
  fixed_property_keys=fixed_prop_keys,
60
60
  )
61
61
  self.p = p
62
- self.peel_kind = f"Shave {p.detection_threshold}"
62
+ self.peel_kind = f"Shave {p.voltage_threshold}"
63
63
  self.denoising_pipeline = denoising_pipeline
64
64
  self.dedup_batch_size = self.nearest_batch_length()
65
65
 
@@ -135,7 +135,7 @@ class Shaver(BasePeeler):
135
135
  channel_index=self.b.channel_index,
136
136
  denoising_pipeline=self.denoising_pipeline,
137
137
  residnorm_decrease_threshold=self.p.shave_score,
138
- detection_threshold=self.p.detection_threshold,
138
+ detection_threshold=self.p.voltage_threshold,
139
139
  peak_sign=self.p.peak_sign,
140
140
  dedup_channel_index=self.b.dedup_channel_index,
141
141
  trough_offset_samples=self.trough_offset_samples,
@@ -12,9 +12,7 @@ import torch
12
12
  from spikeinterface.core import BaseRecording
13
13
 
14
14
  from ..transform import (
15
- BaseTemporalPCA,
16
15
  Voltage,
17
- VQMatchedFilter,
18
16
  Waveform,
19
17
  WaveformPipeline,
20
18
  WaveformWhitener,
@@ -41,7 +39,13 @@ from ..util.waveform_util import (
41
39
  )
42
40
  from .peel_base import BasePeeler, PeelingBatchResult
43
41
  from .peel_lib import threshold_to_fit
44
- from .subtract_util import ChunkSubtracter, make_peak_proposer
42
+ from .subtract_util import (
43
+ ChunkSubtracter,
44
+ make_peak_proposer,
45
+ proposal_filters_from_pipeline,
46
+ proposal_fit_featurization_cfg,
47
+ with_proposal_filters,
48
+ )
45
49
 
46
50
 
47
51
  class SubtractionPeeler(BasePeeler):
@@ -60,6 +64,7 @@ class SubtractionPeeler(BasePeeler):
60
64
  p: SubtractionConfig = default_subtraction_cfg,
61
65
  waveform_cfg: WaveformConfig = default_waveform_cfg,
62
66
  fit_sampling_cfg: FitSamplingConfig = default_peeling_fit_sampling_cfg,
67
+ featurization_cfg: FeaturizationConfig | None = None,
63
68
  save_iteration=False,
64
69
  save_residnorm_decrease=False,
65
70
  save_collidedness=False,
@@ -89,6 +94,8 @@ class SubtractionPeeler(BasePeeler):
89
94
  self.save_iteration = save_iteration
90
95
  self.save_residnorm_decrease = save_residnorm_decrease
91
96
  self.save_collidedness = save_collidedness
97
+ # non-voltage proposal may need this config
98
+ self.featurization_cfg = featurization_cfg
92
99
  self.dedup_batch_size = self.nearest_batch_length()
93
100
  if self.p.whiten:
94
101
  self.threshold = self.p.threshold_before_whitening
@@ -181,8 +188,13 @@ class SubtractionPeeler(BasePeeler):
181
188
  return self._subtracters.subtracter
182
189
 
183
190
  def _build_chunk_subtracter(self) -> ChunkSubtracter:
191
+ proposal = self.p.detection_proposal
192
+ if self.b.proposal_filters is None:
193
+ # hit during the fit pass before we have that gizmo
194
+ proposal = "voltage"
184
195
  proposer = make_peak_proposer(
185
196
  self.p,
197
+ proposal,
186
198
  spike_length_samples=self.spike_length_samples,
187
199
  trough_offset_samples=self.trough_offset_samples,
188
200
  peak_channel_index=self.b.peak_channel_index,
@@ -269,32 +281,18 @@ class SubtractionPeeler(BasePeeler):
269
281
  def _install_proposal_filters(self):
270
282
  assert self.featurization_pipeline is not None
271
283
  assert not self.featurization_pipeline.needs_fit()
272
- if self.p.detection_proposal == "vq":
273
- nodes = [
274
- t
275
- for t in self.featurization_pipeline.transformers
276
- if isinstance(t, VQMatchedFilter)
277
- ]
278
- else:
279
- nodes = [
280
- t
281
- for t in self.featurization_pipeline.transformers
282
- if type(t) is BaseTemporalPCA
283
- ]
284
- if len(nodes) != 1:
285
- panic(
286
- f"detection_proposal={self.p.detection_proposal!r} needs exactly one "
287
- f"proposal model in the featurization pipeline, found {len(nodes)}."
288
- )
289
- node = nodes[0]
290
- filters = node.b.components[: self.p.proposal_filters]
291
- filters = filters.to(self.b.sub_channel_index.device).clone()
284
+ filters, trough_offset = proposal_filters_from_pipeline(
285
+ self.featurization_pipeline,
286
+ self.p.detection_proposal,
287
+ self.p.proposal_filters,
288
+ )
289
+ filters = filters.to(self.b.sub_channel_index.device)
292
290
  if self.b.proposal_filters is None:
293
291
  self.del_none_buffer("proposal_filters")
294
292
  self.register_buffer("proposal_filters", filters)
295
293
  else:
296
294
  self.proposal_filters = filters
297
- self.proposal_filter_trough_offset = node.filter_trough_offset
295
+ self.proposal_filter_trough_offset = trough_offset
298
296
 
299
297
  def save_models(self, save_folder: str | Path):
300
298
  sub_denoise_pt = Path(save_folder) / "subtraction_denoising_pipeline.pt"
@@ -337,14 +335,10 @@ class SubtractionPeeler(BasePeeler):
337
335
  )
338
336
 
339
337
  # handle peak proposal models
340
- proposal = subtraction_cfg.detection_proposal
341
338
  denoising_cfg = subtraction_cfg.subtraction_denoising_cfg
342
- if proposal == "score_net":
339
+ if subtraction_cfg.propose_with_score_net:
343
340
  if not denoising_cfg.score_filter_radius_um:
344
- panic(
345
- "detection_proposal='score_net' needs score_filter_radius_um "
346
- "set in the subtraction denoising config."
347
- )
341
+ panic()
348
342
  else:
349
343
  subtraction_cfg = replace(
350
344
  subtraction_cfg,
@@ -352,19 +346,11 @@ class SubtractionPeeler(BasePeeler):
352
346
  denoising_cfg, score_filter_radius_um=None
353
347
  ),
354
348
  )
355
- if proposal == "tpca":
356
- if subtraction_cfg.proposal_filters > featurization_cfg.tpca_rank:
357
- panic(
358
- f"proposal_filters={subtraction_cfg.proposal_filters} exceeds "
359
- f"tpca_rank={featurization_cfg.tpca_rank}."
360
- )
361
- featurization_cfg = replace(
362
- featurization_cfg, learn_cleaned_tpca_basis=True
363
- )
364
- elif proposal == "vq":
365
- featurization_cfg = replace(
366
- featurization_cfg, vq_proposal_filters=subtraction_cfg.proposal_filters
367
- )
349
+ featurization_cfg = with_proposal_filters(
350
+ featurization_cfg,
351
+ subtraction_cfg.detection_proposal,
352
+ subtraction_cfg.proposal_filters,
353
+ )
368
354
 
369
355
  # construct denoising and featurization pipelines
370
356
  subtraction_denoising_pipeline = WaveformPipeline.from_config(
@@ -402,6 +388,7 @@ class SubtractionPeeler(BasePeeler):
402
388
  p=subtraction_cfg,
403
389
  waveform_cfg=waveform_cfg,
404
390
  fit_sampling_cfg=sampling_cfg,
391
+ featurization_cfg=featurization_cfg,
405
392
  save_iteration=subtraction_cfg.save_iteration,
406
393
  save_residnorm_decrease=subtraction_cfg.save_residnorm_decrease,
407
394
  save_collidedness=save_collidedness,
@@ -641,7 +628,13 @@ class SubtractionPeeler(BasePeeler):
641
628
 
642
629
  def _threshold_to_fit(self, tmp_dir, fit_pipeline, computation_cfg):
643
630
  threshold_cfg = ThresholdingConfig(
644
- detection_threshold=self.p.voltage_threshold,
631
+ detection_proposal=self.p.detection_proposal,
632
+ proposal_filters=self.p.proposal_filters,
633
+ voltage_threshold=self.p.voltage_threshold,
634
+ proposal_threshold=self.p.proposal_threshold,
635
+ spatial_dedup_radius_um=(
636
+ self.p.first_denoiser_spatial_dedup_radius or self.p.subtract_radius_um
637
+ ),
645
638
  peak_sign=self.p.peak_sign,
646
639
  relative_peak_radius_um=self.p.relative_peak_radius_um,
647
640
  relative_peak_radius_samples=self.p.relative_peak_radius_samples,
@@ -665,8 +658,12 @@ class SubtractionPeeler(BasePeeler):
665
658
  recording=self.recording,
666
659
  waveform_cfg=self.waveform_cfg,
667
660
  channel_index=self.b.sub_channel_index,
668
- spatial_dedup_radius=self.p.first_denoiser_spatial_dedup_radius,
669
661
  threshold_cfg=threshold_cfg,
662
+ featurization_cfg=proposal_fit_featurization_cfg(
663
+ self.featurization_cfg,
664
+ self.p.detection_proposal,
665
+ self.p.proposal_filters,
666
+ ),
670
667
  sampling_cfg=sampling_cfg,
671
668
  max_waveforms_fit=self.p.first_denoiser_max_waveforms_fit,
672
669
  n_residual_snips=self.p.first_denoiser_noise_snips,