DARTsort 0.5.27__tar.gz → 0.5.28__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. {dartsort-0.5.27 → dartsort-0.5.28}/PKG-INFO +1 -1
  2. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/config.py +7 -2
  3. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/peel_lib.py +58 -27
  4. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/shaver.py +2 -2
  5. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/subtract.py +42 -45
  6. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/subtract_util.py +127 -52
  7. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/threshold.py +146 -18
  8. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/decollider.py +130 -66
  9. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/matched_filter_net.py +1 -2
  10. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/pipeline.py +19 -1
  11. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/temporal_pca.py +23 -17
  12. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/transform_base.py +5 -25
  13. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/internal_config.py +32 -6
  14. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/preprocess_util.py +26 -17
  15. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/analysis_plots.py +13 -5
  16. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/PKG-INFO +1 -1
  17. {dartsort-0.5.27 → dartsort-0.5.28}/tests/conftest.py +3 -3
  18. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_subtract.py +106 -15
  19. dartsort-0.5.28/tests/test_threshold.py +80 -0
  20. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_transform.py +13 -2
  21. dartsort-0.5.27/tests/test_threshold.py +0 -9
  22. {dartsort-0.5.27 → dartsort-0.5.28}/.github/workflows/ci.yml +0 -0
  23. {dartsort-0.5.27 → dartsort-0.5.28}/.github/workflows/ci_windows.yml +0 -0
  24. {dartsort-0.5.27 → dartsort-0.5.28}/.github/workflows/deploy.yml +0 -0
  25. {dartsort-0.5.27 → dartsort-0.5.28}/.github/workflows/docs.yml +0 -0
  26. {dartsort-0.5.27 → dartsort-0.5.28}/.gitignore +0 -0
  27. {dartsort-0.5.27 → dartsort-0.5.28}/README.md +0 -0
  28. {dartsort-0.5.27 → dartsort-0.5.28}/docs/index.md +0 -0
  29. {dartsort-0.5.27 → dartsort-0.5.28}/docs/main_api.md +0 -0
  30. {dartsort-0.5.27 → dartsort-0.5.28}/docs/toolkit_api.md +0 -0
  31. {dartsort-0.5.27 → dartsort-0.5.28}/environment.yml +0 -0
  32. {dartsort-0.5.27 → dartsort-0.5.28}/notebook/.keep +0 -0
  33. {dartsort-0.5.27 → dartsort-0.5.28}/notebook/fit_and_batch_matching.py +0 -0
  34. {dartsort-0.5.27 → dartsort-0.5.28}/notebook/ibl_dandi_example.py +0 -0
  35. {dartsort-0.5.27 → dartsort-0.5.28}/pyproject.toml +0 -0
  36. {dartsort-0.5.27 → dartsort-0.5.28}/setup.cfg +0 -0
  37. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/__init__.py +0 -0
  38. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/__main__.py +0 -0
  39. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/cli.py +0 -0
  40. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/__init__.py +0 -0
  41. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/agglomerate.py +0 -0
  42. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/cluster_util.py +0 -0
  43. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/clustering.py +0 -0
  44. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/clustering_features.py +0 -0
  45. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/density.py +0 -0
  46. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/forward_backward.py +0 -0
  47. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/kmeans.py +0 -0
  48. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/merge.py +0 -0
  49. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/mixture.py +0 -0
  50. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/refine_util.py +0 -0
  51. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/detect/__init__.py +0 -0
  52. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/detect/detect.py +0 -0
  53. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/__init__.py +0 -0
  54. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/analysis.py +0 -0
  55. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/comparison.py +0 -0
  56. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/hybrid_util.py +0 -0
  57. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/hybridkit.py +0 -0
  58. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  59. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  60. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/simkit.py +0 -0
  61. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/simlib.py +0 -0
  62. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/localize/__init__.py +0 -0
  63. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/localize/localize_torch.py +0 -0
  64. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/localize/localize_util.py +0 -0
  65. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/main.py +0 -0
  66. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/__init__.py +0 -0
  67. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/grab.py +0 -0
  68. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching.py +0 -0
  69. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/__init__.py +0 -0
  70. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  71. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/drifty.py +0 -0
  72. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  73. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  74. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  75. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/peel_base.py +0 -0
  76. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/reduction_template.py +0 -0
  77. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  78. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/detect_np1.pt +0 -0
  79. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/detect_np2.pt +0 -0
  80. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  81. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  82. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  83. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  84. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  85. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  86. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/__init__.py +0 -0
  87. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/get_templates.py +0 -0
  88. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/postprocess_util.py +0 -0
  89. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/realignment.py +0 -0
  90. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/template_util.py +0 -0
  91. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/templates.py +0 -0
  92. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/templib.py +0 -0
  93. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/__init__.py +0 -0
  94. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  95. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/all_transformers.py +0 -0
  96. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/amortized_localization.py +0 -0
  97. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/amplitudes.py +0 -0
  98. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/denoising_scorer.py +0 -0
  99. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/enforce_decrease.py +0 -0
  100. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/fixed_prop.py +0 -0
  101. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/interp.py +0 -0
  102. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/localize.py +0 -0
  103. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/matching_denoiser.py +0 -0
  104. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/mixture_classifier.py +0 -0
  105. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/reduction.py +0 -0
  106. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  107. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/supervised_denoiser.py +0 -0
  108. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/vq_matched_filter.py +0 -0
  109. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/whiten.py +0 -0
  110. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/__init__.py +0 -0
  111. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/cli_util.py +0 -0
  112. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/data_util.py +0 -0
  113. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/drift_util.py +0 -0
  114. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/interpolation_util.py +0 -0
  115. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/job_util.py +0 -0
  116. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/logging_util.py +0 -0
  117. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/main_util.py +0 -0
  118. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/more_operators.py +0 -0
  119. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/motion.py +0 -0
  120. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/multiprocessing_util.py +0 -0
  121. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/nn_util.py +0 -0
  122. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/noise_util.py +0 -0
  123. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/peel_util.py +0 -0
  124. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/py_util.py +0 -0
  125. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/registration_util.py +0 -0
  126. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/residual_util.py +0 -0
  127. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/sparse_util.py +0 -0
  128. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/spikeio.py +0 -0
  129. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/spiketorch.py +0 -0
  130. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/testing_util/__init__.py +0 -0
  131. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  132. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  133. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/torch_optimization_util.py +0 -0
  134. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/torch_util.py +0 -0
  135. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/waveform_util.py +0 -0
  136. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/__init__.py +0 -0
  137. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/colors.py +0 -0
  138. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/gt.py +0 -0
  139. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/layout.py +0 -0
  140. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/metrics_util.py +0 -0
  141. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/mixture.py +0 -0
  142. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/recanim.py +0 -0
  143. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/scatterplots.py +0 -0
  144. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/sorting.py +0 -0
  145. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/unit.py +0 -0
  146. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/unit_comparison.py +0 -0
  147. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/versus.py +0 -0
  148. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/vismain.py +0 -0
  149. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/waveforms.py +0 -0
  150. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/SOURCES.txt +0 -0
  151. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/dependency_links.txt +0 -0
  152. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/entry_points.txt +0 -0
  153. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/requires.txt +0 -0
  154. {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/top_level.txt +0 -0
  155. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_agglomerate.py +0 -0
  156. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_alignment.py +0 -0
  157. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_cluster_util.py +0 -0
  158. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_clustering.py +0 -0
  159. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_clustering_features.py +0 -0
  160. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_config.py +0 -0
  161. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_dartsort.py +0 -0
  162. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_data_util.py +0 -0
  163. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_density.py +0 -0
  164. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_detect.py +0 -0
  165. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_drift_util.py +0 -0
  166. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_drifty_matching.py +0 -0
  167. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_enforce_decrease.py +0 -0
  168. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_grab_and_featurize.py +0 -0
  169. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_hybridkit.py +0 -0
  170. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_kmeans.py +0 -0
  171. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_linalg.py +0 -0
  172. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_localize_torch.py +0 -0
  173. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_matching.py +0 -0
  174. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_mixture_classifier.py +0 -0
  175. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_modeling.py +0 -0
  176. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_simkit.py +0 -0
  177. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_spiketorch.py +0 -0
  178. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_templates.py +0 -0
  179. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_torch_optimization_util.py +0 -0
  180. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_util.py +0 -0
  181. {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_waveform_util.py +0 -0
  182. {dartsort-0.5.27 → dartsort-0.5.28}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.27
3
+ Version: 0.5.28
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -4,6 +4,7 @@ from typing import Annotated, Literal
4
4
  from pydantic import Field
5
5
 
6
6
  from .util.internal_config import (
7
+ DetectionProposal,
7
8
  InterpKernel,
8
9
  InterpMethod,
9
10
  KmeansppSelection,
@@ -12,6 +13,7 @@ from .util.internal_config import (
12
13
  PreprocessingStrategy,
13
14
  RealignStrategy,
14
15
  TemplateSVDMethod,
16
+ TPCADenoiserFitStage,
15
17
  WhiteningEstimator,
16
18
  WhiteningStrategy,
17
19
  default_pretrained_path, # noqa
@@ -277,10 +279,13 @@ class DeveloperConfig(DARTsortUserConfig):
277
279
  nn_denoiser_noise_waveforms: int = 100 * 256
278
280
  nn_denoiser_extra_kwargs: dict | None = None
279
281
  score_filter_radius_um: float = 0.0
280
- detection_proposal: Literal["voltage", "tpca", "vq", "score_net"] = "voltage"
282
+ detection_proposal: DetectionProposal = "voltage"
281
283
  proposal_filters: int = 1
282
- score_proposal_threshold: float = 5.0
284
+ propose_with_score_net: bool = False
285
+ proposal_threshold: float = 5.0
286
+ score_net_threshold: float = 5.0
283
287
  do_tpca_denoise: bool = True
288
+ tpca_denoise_fit: TPCADenoiserFitStage | None = None
284
289
  first_denoiser_thinning: float = 0.0
285
290
  first_denoiser_spatial_dedup_radius: float = 100.0
286
291
  realign_to_denoiser: bool = True
@@ -12,6 +12,7 @@ from torch import Tensor
12
12
  from ..detect import detect_and_deduplicate
13
13
  from ..util.internal_config import (
14
14
  ComputationConfig,
15
+ FeaturizationConfig,
15
16
  FitSamplingConfig,
16
17
  PeakSign,
17
18
  ThresholdingConfig,
@@ -20,7 +21,6 @@ from ..util.internal_config import (
20
21
  from ..util.job_util import ensure_computation_config
21
22
  from ..util.spiketorch import grab_spikes, subtract_spikes_
22
23
  from ..util.torch_util import torch_compile
23
- from ..util.waveform_util import make_channel_index
24
24
  from .peel_base import PeelingBatchResult
25
25
 
26
26
  if TYPE_CHECKING:
@@ -120,7 +120,7 @@ def flatten_denan_and_whiten_batched(
120
120
  def threshold_chunk(
121
121
  traces,
122
122
  channel_index,
123
- detection_threshold=4.0,
123
+ voltage_threshold=4.0,
124
124
  peak_sign: PeakSign = "both",
125
125
  peak_channel_index=None,
126
126
  dedup_channel_index=None,
@@ -140,10 +140,9 @@ def threshold_chunk(
140
140
  rg=None,
141
141
  quiet=False,
142
142
  ) -> PeelingBatchResult:
143
- n_index = channel_index.shape[1]
144
143
  times_rel, channels, energies = detect_and_deduplicate(
145
144
  traces,
146
- threshold=detection_threshold,
145
+ threshold=voltage_threshold,
147
146
  peak_channel_index=peak_channel_index,
148
147
  dedup_neighborhoods=dedup_channel_index,
149
148
  peak_sign=peak_sign,
@@ -153,6 +152,46 @@ def threshold_chunk(
153
152
  return_energies=True,
154
153
  trough_priority=trough_priority,
155
154
  )
155
+ return extract_detections(
156
+ traces,
157
+ times_rel,
158
+ channels,
159
+ energies,
160
+ channel_index=channel_index,
161
+ trough_offset_samples=trough_offset_samples,
162
+ spike_length_samples=spike_length_samples,
163
+ left_margin=left_margin,
164
+ right_margin=right_margin,
165
+ max_spikes_per_chunk=max_spikes_per_chunk,
166
+ thinning=thinning,
167
+ time_jitter=time_jitter,
168
+ spatial_jitter_channel_index=spatial_jitter_channel_index,
169
+ return_waveforms=return_waveforms,
170
+ rg=rg,
171
+ quiet=quiet,
172
+ )
173
+
174
+
175
+ def extract_detections(
176
+ traces,
177
+ times_rel,
178
+ channels,
179
+ energies,
180
+ *,
181
+ channel_index,
182
+ trough_offset_samples=42,
183
+ spike_length_samples=121,
184
+ left_margin=0,
185
+ right_margin=0,
186
+ max_spikes_per_chunk=None,
187
+ thinning=0.0,
188
+ time_jitter=0,
189
+ spatial_jitter_channel_index=None,
190
+ return_waveforms=True,
191
+ rg=None,
192
+ quiet=False,
193
+ ) -> PeelingBatchResult:
194
+ n_index = channel_index.shape[1]
156
195
  if not times_rel.numel():
157
196
  return PeelingBatchResult(
158
197
  n_spikes=0,
@@ -281,7 +320,7 @@ def perturb_detections(
281
320
  n = len(times_rel)
282
321
  if time_jitter:
283
322
  assert rg is not None
284
- jitter = rg.integers(low=-time_jitter, high=time_jitter + 1)
323
+ jitter = rg.integers(low=-time_jitter, high=time_jitter + 1, size=n)
285
324
  times_rel = times_rel + torch.asarray(
286
325
  jitter, dtype=times_rel.dtype, device=times_rel.device
287
326
  )
@@ -436,8 +475,8 @@ def threshold_to_fit(
436
475
  recording: BaseRecording,
437
476
  waveform_cfg: WaveformConfig,
438
477
  channel_index: Tensor,
439
- spatial_dedup_radius: float | None,
440
478
  threshold_cfg: ThresholdingConfig,
479
+ featurization_cfg: FeaturizationConfig,
441
480
  sampling_cfg: FitSamplingConfig,
442
481
  max_waveforms_fit: int | None = None,
443
482
  n_residual_snips: int | None = None,
@@ -448,34 +487,23 @@ def threshold_to_fit(
448
487
 
449
488
  Used by subtraction to fit initial NN denoisers.
450
489
  """
451
- from ..transform import Waveform, WaveformPipeline
452
490
  from ..util.data_util import subsample_waveforms
453
491
  from .threshold import Threshold
454
492
 
455
493
  computation_cfg = ensure_computation_config(computation_cfg)
456
494
 
457
- geom = recording.get_channel_locations()
458
- waveform_node = Waveform(
459
- channel_index=channel_index,
460
- waveform_cfg=waveform_cfg,
461
- sampling_frequency=recording.sampling_frequency,
462
- )
463
- waveform_pipeline = WaveformPipeline([waveform_node])
464
-
465
- if spatial_dedup_radius:
466
- dn_dedup_ci = make_channel_index(geom, spatial_dedup_radius, to_torch=True)
467
- dn_dedup_ci = dn_dedup_ci.to(channel_index)
468
- else:
469
- dn_dedup_ci = channel_index
470
- trainer = Threshold(
495
+ trainer = Threshold.from_config(
471
496
  recording=recording,
472
- channel_index=channel_index,
473
- featurization_pipeline=waveform_pipeline,
474
- p=threshold_cfg,
475
497
  waveform_cfg=waveform_cfg,
476
- dedup_channel_index=dn_dedup_ci,
477
- fit_sampling_cfg=sampling_cfg,
498
+ thresholding_cfg=threshold_cfg,
499
+ featurization_cfg=featurization_cfg,
500
+ sampling_cfg=sampling_cfg,
501
+ extract_channel_index=channel_index,
478
502
  )
503
+ assert trainer.featurization_pipeline is not None
504
+ assert "waveforms" in {
505
+ ds.name for ds in trainer.featurization_pipeline.spike_datasets()
506
+ }
479
507
 
480
508
  if max_waveforms_fit is None:
481
509
  max_waveforms_fit = sampling_cfg.max_waveforms_fit
@@ -490,6 +518,9 @@ def threshold_to_fit(
490
518
  with TemporaryDirectory(dir=tmp_dir) as temp_dir:
491
519
  temp_hdf5_filename = Path(temp_dir) / "subtraction_denoiser0_fit.h5"
492
520
  try:
521
+ trainer.load_or_fit_and_save_models(
522
+ Path(temp_dir) / "threshold_models", computation_cfg=computation_cfg
523
+ )
493
524
  trainer.run_subsampled_peeling(
494
525
  temp_hdf5_filename,
495
526
  stop_after_n_waveforms=max_waveforms_fit,
@@ -515,7 +546,7 @@ def threshold_to_fit(
515
546
  raise ValueError(
516
547
  "Found no spikes when thresholding to get model fitting data. "
517
548
  "This usually indicates a preprocessing issue, since it means "
518
- f"that no spikes could be found at threshold {threshold_cfg.detection_threshold}."
549
+ f"that no spikes could be found at threshold {threshold_cfg.voltage_threshold}."
519
550
  )
520
551
 
521
552
  # fit the thing
@@ -59,7 +59,7 @@ class Shaver(BasePeeler):
59
59
  fixed_property_keys=fixed_prop_keys,
60
60
  )
61
61
  self.p = p
62
- self.peel_kind = f"Shave {p.detection_threshold}"
62
+ self.peel_kind = f"Shave {p.voltage_threshold}"
63
63
  self.denoising_pipeline = denoising_pipeline
64
64
  self.dedup_batch_size = self.nearest_batch_length()
65
65
 
@@ -135,7 +135,7 @@ class Shaver(BasePeeler):
135
135
  channel_index=self.b.channel_index,
136
136
  denoising_pipeline=self.denoising_pipeline,
137
137
  residnorm_decrease_threshold=self.p.shave_score,
138
- detection_threshold=self.p.detection_threshold,
138
+ detection_threshold=self.p.voltage_threshold,
139
139
  peak_sign=self.p.peak_sign,
140
140
  dedup_channel_index=self.b.dedup_channel_index,
141
141
  trough_offset_samples=self.trough_offset_samples,
@@ -12,9 +12,7 @@ import torch
12
12
  from spikeinterface.core import BaseRecording
13
13
 
14
14
  from ..transform import (
15
- BaseTemporalPCA,
16
15
  Voltage,
17
- VQMatchedFilter,
18
16
  Waveform,
19
17
  WaveformPipeline,
20
18
  WaveformWhitener,
@@ -41,7 +39,13 @@ from ..util.waveform_util import (
41
39
  )
42
40
  from .peel_base import BasePeeler, PeelingBatchResult
43
41
  from .peel_lib import threshold_to_fit
44
- from .subtract_util import ChunkSubtracter, make_peak_proposer
42
+ from .subtract_util import (
43
+ ChunkSubtracter,
44
+ make_peak_proposer,
45
+ proposal_filters_from_pipeline,
46
+ proposal_fit_featurization_cfg,
47
+ with_proposal_filters,
48
+ )
45
49
 
46
50
 
47
51
  class SubtractionPeeler(BasePeeler):
@@ -60,6 +64,7 @@ class SubtractionPeeler(BasePeeler):
60
64
  p: SubtractionConfig = default_subtraction_cfg,
61
65
  waveform_cfg: WaveformConfig = default_waveform_cfg,
62
66
  fit_sampling_cfg: FitSamplingConfig = default_peeling_fit_sampling_cfg,
67
+ featurization_cfg: FeaturizationConfig | None = None,
63
68
  save_iteration=False,
64
69
  save_residnorm_decrease=False,
65
70
  save_collidedness=False,
@@ -89,6 +94,8 @@ class SubtractionPeeler(BasePeeler):
89
94
  self.save_iteration = save_iteration
90
95
  self.save_residnorm_decrease = save_residnorm_decrease
91
96
  self.save_collidedness = save_collidedness
97
+ # non-voltage proposal may need this config
98
+ self.featurization_cfg = featurization_cfg
92
99
  self.dedup_batch_size = self.nearest_batch_length()
93
100
  if self.p.whiten:
94
101
  self.threshold = self.p.threshold_before_whitening
@@ -181,8 +188,13 @@ class SubtractionPeeler(BasePeeler):
181
188
  return self._subtracters.subtracter
182
189
 
183
190
  def _build_chunk_subtracter(self) -> ChunkSubtracter:
191
+ proposal = self.p.detection_proposal
192
+ if self.b.proposal_filters is None:
193
+ # hit during the fit pass before we have that gizmo
194
+ proposal = "voltage"
184
195
  proposer = make_peak_proposer(
185
196
  self.p,
197
+ proposal,
186
198
  spike_length_samples=self.spike_length_samples,
187
199
  trough_offset_samples=self.trough_offset_samples,
188
200
  peak_channel_index=self.b.peak_channel_index,
@@ -269,32 +281,18 @@ class SubtractionPeeler(BasePeeler):
269
281
  def _install_proposal_filters(self):
270
282
  assert self.featurization_pipeline is not None
271
283
  assert not self.featurization_pipeline.needs_fit()
272
- if self.p.detection_proposal == "vq":
273
- nodes = [
274
- t
275
- for t in self.featurization_pipeline.transformers
276
- if isinstance(t, VQMatchedFilter)
277
- ]
278
- else:
279
- nodes = [
280
- t
281
- for t in self.featurization_pipeline.transformers
282
- if type(t) is BaseTemporalPCA
283
- ]
284
- if len(nodes) != 1:
285
- panic(
286
- f"detection_proposal={self.p.detection_proposal!r} needs exactly one "
287
- f"proposal model in the featurization pipeline, found {len(nodes)}."
288
- )
289
- node = nodes[0]
290
- filters = node.b.components[: self.p.proposal_filters]
291
- filters = filters.to(self.b.sub_channel_index.device).clone()
284
+ filters, trough_offset = proposal_filters_from_pipeline(
285
+ self.featurization_pipeline,
286
+ self.p.detection_proposal,
287
+ self.p.proposal_filters,
288
+ )
289
+ filters = filters.to(self.b.sub_channel_index.device)
292
290
  if self.b.proposal_filters is None:
293
291
  self.del_none_buffer("proposal_filters")
294
292
  self.register_buffer("proposal_filters", filters)
295
293
  else:
296
294
  self.proposal_filters = filters
297
- self.proposal_filter_trough_offset = node.filter_trough_offset
295
+ self.proposal_filter_trough_offset = trough_offset
298
296
 
299
297
  def save_models(self, save_folder: str | Path):
300
298
  sub_denoise_pt = Path(save_folder) / "subtraction_denoising_pipeline.pt"
@@ -337,14 +335,10 @@ class SubtractionPeeler(BasePeeler):
337
335
  )
338
336
 
339
337
  # handle peak proposal models
340
- proposal = subtraction_cfg.detection_proposal
341
338
  denoising_cfg = subtraction_cfg.subtraction_denoising_cfg
342
- if proposal == "score_net":
339
+ if subtraction_cfg.propose_with_score_net:
343
340
  if not denoising_cfg.score_filter_radius_um:
344
- panic(
345
- "detection_proposal='score_net' needs score_filter_radius_um "
346
- "set in the subtraction denoising config."
347
- )
341
+ panic()
348
342
  else:
349
343
  subtraction_cfg = replace(
350
344
  subtraction_cfg,
@@ -352,19 +346,11 @@ class SubtractionPeeler(BasePeeler):
352
346
  denoising_cfg, score_filter_radius_um=None
353
347
  ),
354
348
  )
355
- if proposal == "tpca":
356
- if subtraction_cfg.proposal_filters > featurization_cfg.tpca_rank:
357
- panic(
358
- f"proposal_filters={subtraction_cfg.proposal_filters} exceeds "
359
- f"tpca_rank={featurization_cfg.tpca_rank}."
360
- )
361
- featurization_cfg = replace(
362
- featurization_cfg, learn_cleaned_tpca_basis=True
363
- )
364
- elif proposal == "vq":
365
- featurization_cfg = replace(
366
- featurization_cfg, vq_proposal_filters=subtraction_cfg.proposal_filters
367
- )
349
+ featurization_cfg = with_proposal_filters(
350
+ featurization_cfg,
351
+ subtraction_cfg.detection_proposal,
352
+ subtraction_cfg.proposal_filters,
353
+ )
368
354
 
369
355
  # construct denoising and featurization pipelines
370
356
  subtraction_denoising_pipeline = WaveformPipeline.from_config(
@@ -402,6 +388,7 @@ class SubtractionPeeler(BasePeeler):
402
388
  p=subtraction_cfg,
403
389
  waveform_cfg=waveform_cfg,
404
390
  fit_sampling_cfg=sampling_cfg,
391
+ featurization_cfg=featurization_cfg,
405
392
  save_iteration=subtraction_cfg.save_iteration,
406
393
  save_residnorm_decrease=subtraction_cfg.save_residnorm_decrease,
407
394
  save_collidedness=save_collidedness,
@@ -641,7 +628,13 @@ class SubtractionPeeler(BasePeeler):
641
628
 
642
629
  def _threshold_to_fit(self, tmp_dir, fit_pipeline, computation_cfg):
643
630
  threshold_cfg = ThresholdingConfig(
644
- detection_threshold=self.p.voltage_threshold,
631
+ detection_proposal=self.p.detection_proposal,
632
+ proposal_filters=self.p.proposal_filters,
633
+ voltage_threshold=self.p.voltage_threshold,
634
+ proposal_threshold=self.p.proposal_threshold,
635
+ spatial_dedup_radius_um=(
636
+ self.p.first_denoiser_spatial_dedup_radius or self.p.subtract_radius_um
637
+ ),
645
638
  peak_sign=self.p.peak_sign,
646
639
  relative_peak_radius_um=self.p.relative_peak_radius_um,
647
640
  relative_peak_radius_samples=self.p.relative_peak_radius_samples,
@@ -665,8 +658,12 @@ class SubtractionPeeler(BasePeeler):
665
658
  recording=self.recording,
666
659
  waveform_cfg=self.waveform_cfg,
667
660
  channel_index=self.b.sub_channel_index,
668
- spatial_dedup_radius=self.p.first_denoiser_spatial_dedup_radius,
669
661
  threshold_cfg=threshold_cfg,
662
+ featurization_cfg=proposal_fit_featurization_cfg(
663
+ self.featurization_cfg,
664
+ self.p.detection_proposal,
665
+ self.p.proposal_filters,
666
+ ),
670
667
  sampling_cfg=sampling_cfg,
671
668
  max_waveforms_fit=self.p.first_denoiser_max_waveforms_fit,
672
669
  n_residual_snips=self.p.first_denoiser_noise_snips,
@@ -1,3 +1,4 @@
1
+ from dataclasses import replace
1
2
  from typing import TYPE_CHECKING, Literal, Protocol, Self
2
3
 
3
4
  import torch
@@ -11,7 +12,13 @@ from ..detect.detect import (
11
12
  peak_sign_to_pos,
12
13
  update_peak_map,
13
14
  )
14
- from ..util.internal_config import PeakSign, SubtractionConfig
15
+ from ..transform import BaseTemporalPCA, VQMatchedFilter
16
+ from ..util.internal_config import (
17
+ DetectionProposal,
18
+ FeaturizationConfig,
19
+ PeakSign,
20
+ SubtractionConfig,
21
+ )
15
22
  from ..util.py_util import databag, panic
16
23
  from ..util.spiketorch import grab_spikes, subtract_spikes_
17
24
  from ..util.torch_util import BModule
@@ -165,7 +172,9 @@ class GlobalPeakProposer:
165
172
  reference = self._full_peak_map(residual)
166
173
  n_bad = int((reference != self.peak_map).sum())
167
174
  if n_bad:
168
- panic(f"patched peak map differs from full recompute at {n_bad} sites")
175
+ raise ValueError(
176
+ f"patched peak map differs from full recompute at {n_bad} sites"
177
+ )
169
178
 
170
179
  def propose_peaks(
171
180
  self, residual: Tensor, detection_mask: Tensor | None
@@ -407,10 +416,14 @@ class MatchedFilterProposer:
407
416
  bad = int(off.sum())
408
417
  if bad:
409
418
  worst = (reference - self.field).abs().nan_to_num().max()
410
- panic(f"patched score field differs at {bad} sites, worst {worst:.3e}")
419
+ raise ValueError(
420
+ f"patched score field differs at {bad} sites, worst {worst:.3e}"
421
+ )
411
422
  bad = int((self._full_peak_map() != self.peak_map).sum())
412
423
  if bad:
413
- panic(f"patched score peak map differs from full recompute at {bad} sites")
424
+ raise ValueError(
425
+ f"patched score peak map differs from full recompute at {bad} sites"
426
+ )
414
427
 
415
428
 
416
429
  @databag
@@ -455,7 +468,7 @@ class LinearMatchedFilterProposer:
455
468
  return self.spike_length_samples + nbefore + nafter
456
469
 
457
470
  def setup(self, residual: Tensor) -> None:
458
- self.field = self._full_field(residual)
471
+ self.field = self.full_proposal_score_field(residual)
459
472
  self.peak_map = self._full_peak_map()
460
473
 
461
474
  def cleanup(self) -> None:
@@ -522,6 +535,12 @@ class LinearMatchedFilterProposer:
522
535
  out[i0:i1] = self.score_from_responses(responses).T
523
536
  return out
524
537
 
538
+ def full_proposal_score_field(self, residual: Tensor) -> Tensor:
539
+ field = residual.new_full(residual.shape, -torch.inf)
540
+ dense = self.dense(residual[:, :-1])
541
+ field[self.trough_offset : self.trough_offset + dense.shape[0], :-1] = dense
542
+ return field
543
+
525
544
  # -- the patch
526
545
 
527
546
  def _patch_field(self, residual: Tensor, peaks: AcceptedPeaks) -> Tensor:
@@ -557,12 +576,6 @@ class LinearMatchedFilterProposer:
557
576
 
558
577
  # -- for audit
559
578
 
560
- def _full_field(self, residual: Tensor) -> Tensor:
561
- field = residual.new_full(residual.shape, -torch.inf)
562
- dense = self.dense(residual[:, :-1])
563
- field[self.trough_offset : self.trough_offset + dense.shape[0], :-1] = dense
564
- return field
565
-
566
579
  def _full_peak_map(self) -> Tensor:
567
580
  assert self.field is not None
568
581
  return is_extreme_transpose_no_pad(
@@ -573,17 +586,21 @@ class LinearMatchedFilterProposer:
573
586
 
574
587
  def _check(self, residual: Tensor) -> None:
575
588
  assert self.field is not None and self.peak_map is not None
576
- reference = self._full_field(residual)
589
+ reference = self.full_proposal_score_field(residual)
577
590
  off = reference.isfinite().logical_and_(
578
591
  torch.isclose(reference, self.field, rtol=1e-4, atol=1e-5).logical_not_()
579
592
  )
580
593
  bad = int(off.sum())
581
594
  if bad:
582
595
  worst = (reference - self.field).abs().nan_to_num().max()
583
- panic(f"patched linear field differs at {bad} sites, worst {worst:.3e}")
596
+ raise ValueError(
597
+ f"patched linear field differs at {bad} sites, worst {worst:.3e}"
598
+ )
584
599
  bad = int((self._full_peak_map() != self.peak_map).sum())
585
600
  if bad:
586
- panic(f"patched linear peak map differs from full recompute at {bad} sites")
601
+ raise ValueError(
602
+ f"patched linear peak map differs from full recompute at {bad} sites"
603
+ )
587
604
 
588
605
 
589
606
  def find_score_net(pipeline: "WaveformPipeline"):
@@ -596,6 +613,7 @@ def find_score_net(pipeline: "WaveformPipeline"):
596
613
 
597
614
  def make_peak_proposer(
598
615
  p: SubtractionConfig,
616
+ proposal: DetectionProposal,
599
617
  spike_length_samples: int,
600
618
  trough_offset_samples: int,
601
619
  peak_channel_index: Tensor | None,
@@ -606,41 +624,16 @@ def make_peak_proposer(
606
624
  proposal_filter_trough_offset: int | None = None,
607
625
  audit: bool = False,
608
626
  ) -> PeakProposer:
609
- if p.detection_proposal not in ("voltage", "tpca", "vq", "score_net"):
610
- panic(f"unknown {p.detection_proposal=}")
611
-
612
- if p.detection_proposal in ("tpca", "vq") and proposal_filters is not None:
613
- assert channel_index is not None
614
- assert proposal_filter_trough_offset is not None
615
- return LinearMatchedFilterProposer(
616
- filters=proposal_filters,
617
- filter_trough_offset=proposal_filter_trough_offset,
618
- threshold=p.score_proposal_threshold**2,
619
- reduction="sum" if p.detection_proposal == "tpca" else "max",
620
- relative_peak_radius=p.relative_peak_radius_samples,
621
- detect_dedup_radius=spike_length_samples,
622
- peak_channel_index=peak_channel_index,
623
- remove_exact_duplicates=p.remove_exact_duplicates,
624
- trough_offset_samples=trough_offset_samples,
625
- spike_length_samples=spike_length_samples,
626
- field_patch_channel_index=channel_index,
627
- audit=audit,
628
- )
629
-
630
- if p.detection_proposal == "score_net":
627
+ if p.propose_with_score_net:
631
628
  assert denoising_pipeline is not None and channel_index is not None
632
629
  denoiser, score_net = find_score_net(denoising_pipeline)
633
- if score_net is None:
634
- panic(
635
- "detection_proposal='score_net' but the denoising pipeline has no "
636
- "score net. Set score_filter_radius_um so one gets trained."
637
- )
630
+ assert score_net is not None
638
631
  score_channel_index = denoiser.b.score_channel_index
639
632
  return MatchedFilterProposer(
640
633
  score_net=score_net,
641
634
  score_channel_index=score_channel_index,
642
635
  score_mask=score_channel_index < denoiser.n_channels,
643
- threshold=p.score_proposal_threshold,
636
+ threshold=p.score_net_threshold,
644
637
  relative_peak_radius=p.relative_peak_radius_samples,
645
638
  detect_dedup_radius=spike_length_samples,
646
639
  peak_channel_index=peak_channel_index,
@@ -652,7 +645,8 @@ def make_peak_proposer(
652
645
  ),
653
646
  audit=audit,
654
647
  )
655
- if p.subtract_global_dedup:
648
+
649
+ if proposal == "voltage" and p.subtract_global_dedup:
656
650
  return GlobalPeakProposer(
657
651
  threshold=p.voltage_threshold,
658
652
  peak_sign=p.peak_sign,
@@ -665,16 +659,97 @@ def make_peak_proposer(
665
659
  spike_length_samples=spike_length_samples,
666
660
  audit=audit,
667
661
  )
668
- return LocalPeakProposer(
669
- threshold=p.voltage_threshold,
670
- peak_sign=p.peak_sign,
671
- relative_peak_radius=p.relative_peak_radius_samples,
672
- detect_dedup_radius=spike_length_samples,
673
- peak_channel_index=peak_channel_index,
674
- sub_dedup_channel_index=sub_dedup_channel_index,
675
- trough_priority=p.trough_priority,
676
- remove_exact_duplicates=p.remove_exact_duplicates,
662
+ elif proposal == "voltage":
663
+ return LocalPeakProposer(
664
+ threshold=p.voltage_threshold,
665
+ peak_sign=p.peak_sign,
666
+ relative_peak_radius=p.relative_peak_radius_samples,
667
+ detect_dedup_radius=spike_length_samples,
668
+ peak_channel_index=peak_channel_index,
669
+ sub_dedup_channel_index=sub_dedup_channel_index,
670
+ trough_priority=p.trough_priority,
671
+ remove_exact_duplicates=p.remove_exact_duplicates,
672
+ )
673
+ elif proposal == "tpca" or proposal == "vq":
674
+ assert proposal_filters is not None
675
+ assert channel_index is not None
676
+ assert proposal_filter_trough_offset is not None
677
+ return LinearMatchedFilterProposer(
678
+ filters=proposal_filters,
679
+ filter_trough_offset=proposal_filter_trough_offset,
680
+ threshold=p.proposal_threshold**2,
681
+ reduction="sum" if proposal == "tpca" else "max",
682
+ relative_peak_radius=p.relative_peak_radius_samples,
683
+ detect_dedup_radius=spike_length_samples,
684
+ peak_channel_index=peak_channel_index,
685
+ remove_exact_duplicates=p.remove_exact_duplicates,
686
+ trough_offset_samples=trough_offset_samples,
687
+ spike_length_samples=spike_length_samples,
688
+ field_patch_channel_index=channel_index,
689
+ audit=audit,
690
+ )
691
+ else:
692
+ panic(proposal)
693
+
694
+
695
+ def with_proposal_filters(
696
+ featurization_cfg: FeaturizationConfig, proposal: DetectionProposal, n_filters: int
697
+ ) -> FeaturizationConfig:
698
+ """Swap fields to make a featurization config whose pipeline will fit the proposer"""
699
+ if proposal == "voltage":
700
+ return featurization_cfg
701
+ elif proposal == "tpca":
702
+ if n_filters > featurization_cfg.tpca_rank:
703
+ panic(f"{n_filters} {featurization_cfg.tpca_rank}")
704
+ return replace(featurization_cfg, learn_cleaned_tpca_basis=True)
705
+ elif proposal == "vq":
706
+ return replace(featurization_cfg, vq_proposal_filters=n_filters)
707
+ else:
708
+ panic(proposal)
709
+
710
+
711
+ def proposal_fit_featurization_cfg(
712
+ featurization_cfg: FeaturizationConfig | None,
713
+ proposal: DetectionProposal,
714
+ n_filters: int,
715
+ ) -> FeaturizationConfig:
716
+ if featurization_cfg is None and proposal != "voltage":
717
+ raise ValueError(
718
+ f"{proposal=} needs the featurization config its filters are fit with."
719
+ )
720
+ if featurization_cfg is None:
721
+ featurization_cfg = FeaturizationConfig()
722
+ waveforms_only = FeaturizationConfig(
723
+ do_tpca_denoise=False,
724
+ do_enforce_decrease="no",
725
+ save_input_voltages=False,
726
+ save_input_waveforms=True,
727
+ save_input_tpca_projs=False,
728
+ save_amplitudes=False,
729
+ do_localization=False,
730
+ input_waveforms_name="",
731
+ tpca_rank=featurization_cfg.tpca_rank,
732
+ tpca_fit_radius=featurization_cfg.tpca_fit_radius,
733
+ tpca_max_waveforms=featurization_cfg.tpca_max_waveforms,
734
+ input_tpca_waveform_cfg=featurization_cfg.input_tpca_waveform_cfg,
677
735
  )
736
+ return with_proposal_filters(waveforms_only, proposal, n_filters)
737
+
738
+
739
+ def proposal_filters_from_pipeline(
740
+ pipeline: "WaveformPipeline", proposal: DetectionProposal, n_filters: int
741
+ ) -> tuple[Tensor, int]:
742
+ if proposal == "tpca":
743
+ nodes = [t for t in pipeline.transformers if type(t) is BaseTemporalPCA]
744
+ elif proposal == "vq":
745
+ nodes = [t for t in pipeline.transformers if isinstance(t, VQMatchedFilter)]
746
+ else:
747
+ panic()
748
+ if len(nodes) != 1:
749
+ panic(len(nodes))
750
+ (node,) = nodes
751
+ assert not node.needs_fit()
752
+ return node.b.components[:n_filters].clone(), node.filter_trough_offset
678
753
 
679
754
 
680
755
  # -- the subtracter