DARTsort 0.5.13__tar.gz → 0.5.14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. {dartsort-0.5.13 → dartsort-0.5.14}/PKG-INFO +1 -1
  2. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/mixture.py +3 -1
  3. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/config.py +3 -3
  4. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/comparison.py +6 -1
  5. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/peel_base.py +11 -1
  6. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/peel_lib.py +64 -32
  7. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/_multichan_denoiser_kit.py +109 -34
  8. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/amplitudes.py +13 -6
  9. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/decollider.py +18 -8
  10. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/pipeline.py +12 -6
  11. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/internal_config.py +5 -4
  12. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/main_util.py +2 -1
  13. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/peel_util.py +16 -0
  14. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/PKG-INFO +1 -1
  15. {dartsort-0.5.13 → dartsort-0.5.14}/.github/workflows/ci.yml +0 -0
  16. {dartsort-0.5.13 → dartsort-0.5.14}/.github/workflows/ci_windows.yml +0 -0
  17. {dartsort-0.5.13 → dartsort-0.5.14}/.github/workflows/deploy.yml +0 -0
  18. {dartsort-0.5.13 → dartsort-0.5.14}/.github/workflows/docs.yml +0 -0
  19. {dartsort-0.5.13 → dartsort-0.5.14}/.gitignore +0 -0
  20. {dartsort-0.5.13 → dartsort-0.5.14}/README.md +0 -0
  21. {dartsort-0.5.13 → dartsort-0.5.14}/docs/index.md +0 -0
  22. {dartsort-0.5.13 → dartsort-0.5.14}/docs/main_api.md +0 -0
  23. {dartsort-0.5.13 → dartsort-0.5.14}/docs/toolkit_api.md +0 -0
  24. {dartsort-0.5.13 → dartsort-0.5.14}/environment.yml +0 -0
  25. {dartsort-0.5.13 → dartsort-0.5.14}/notebook/.keep +0 -0
  26. {dartsort-0.5.13 → dartsort-0.5.14}/pyproject.toml +0 -0
  27. {dartsort-0.5.13 → dartsort-0.5.14}/setup.cfg +0 -0
  28. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/__init__.py +0 -0
  29. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/cli.py +0 -0
  30. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/__init__.py +0 -0
  31. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/agglomerate.py +0 -0
  32. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/cluster_util.py +0 -0
  33. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/clustering.py +0 -0
  34. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/clustering_features.py +0 -0
  35. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/density.py +0 -0
  36. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/forward_backward.py +0 -0
  37. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/kmeans.py +0 -0
  38. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/merge.py +0 -0
  39. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/modes.py +0 -0
  40. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/refine_util.py +0 -0
  41. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/detect/__init__.py +0 -0
  42. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/detect/detect.py +0 -0
  43. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/detect/detect_filters.py +0 -0
  44. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/__init__.py +0 -0
  45. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/analysis.py +0 -0
  46. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/hybrid_util.py +0 -0
  47. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  48. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  49. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/simkit.py +0 -0
  50. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/simlib.py +0 -0
  51. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/localize/__init__.py +0 -0
  52. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/localize/localize_torch.py +0 -0
  53. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/localize/localize_util.py +0 -0
  54. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/main.py +0 -0
  55. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/__init__.py +0 -0
  56. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/grab.py +0 -0
  57. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching.py +0 -0
  58. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/__init__.py +0 -0
  59. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  60. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/drifty.py +0 -0
  61. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  62. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  63. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  64. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/reduction_template.py +0 -0
  65. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/shaver.py +0 -0
  66. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/subtract.py +0 -0
  67. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/threshold.py +0 -0
  68. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  69. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/detect_np1.pt +0 -0
  70. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/detect_np2.pt +0 -0
  71. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  72. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  73. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  74. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  75. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  76. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  77. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/__init__.py +0 -0
  78. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/get_templates.py +0 -0
  79. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/postprocess_util.py +0 -0
  80. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/realignment.py +0 -0
  81. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/template_util.py +0 -0
  82. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/templates.py +0 -0
  83. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/templib.py +0 -0
  84. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/__init__.py +0 -0
  85. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/all_transformers.py +0 -0
  86. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/amortized_localization.py +0 -0
  87. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/denoising_scorer.py +0 -0
  88. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/enforce_decrease.py +0 -0
  89. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/fixed_prop.py +0 -0
  90. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/interp.py +0 -0
  91. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/localize.py +0 -0
  92. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/matching_denoiser.py +0 -0
  93. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/mixture_classifier.py +0 -0
  94. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/reduction.py +0 -0
  95. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  96. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/supervised_denoiser.py +0 -0
  97. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/temporal_pca.py +0 -0
  98. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/transform_base.py +0 -0
  99. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/whiten.py +0 -0
  100. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/__init__.py +0 -0
  101. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/cli_util.py +0 -0
  102. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/data_util.py +0 -0
  103. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/drift_util.py +0 -0
  104. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/interpolation_util.py +0 -0
  105. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/job_util.py +0 -0
  106. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/logging_util.py +0 -0
  107. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/more_operators.py +0 -0
  108. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/motion.py +0 -0
  109. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/multiprocessing_util.py +0 -0
  110. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/nn_util.py +0 -0
  111. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/noise_util.py +0 -0
  112. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/preprocess_util.py +0 -0
  113. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/py_util.py +0 -0
  114. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/registration_util.py +0 -0
  115. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/residual_util.py +0 -0
  116. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/sparse_util.py +0 -0
  117. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/spikeio.py +0 -0
  118. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/spiketorch.py +0 -0
  119. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/testing_util/__init__.py +0 -0
  120. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  121. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  122. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/torch_optimization_util.py +0 -0
  123. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/torch_util.py +0 -0
  124. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/waveform_util.py +0 -0
  125. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/__init__.py +0 -0
  126. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/analysis_plots.py +0 -0
  127. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/colors.py +0 -0
  128. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/gt.py +0 -0
  129. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/layout.py +0 -0
  130. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/metrics_util.py +0 -0
  131. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/mixture.py +0 -0
  132. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/recanim.py +0 -0
  133. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/scatterplots.py +0 -0
  134. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/sorting.py +0 -0
  135. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/unit.py +0 -0
  136. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/unit_comparison.py +0 -0
  137. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/versus.py +0 -0
  138. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/vismain.py +0 -0
  139. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/waveforms.py +0 -0
  140. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/SOURCES.txt +0 -0
  141. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/dependency_links.txt +0 -0
  142. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/entry_points.txt +0 -0
  143. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/requires.txt +0 -0
  144. {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/top_level.txt +0 -0
  145. {dartsort-0.5.13 → dartsort-0.5.14}/tests/conftest.py +0 -0
  146. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_agglomerate.py +0 -0
  147. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_alignment.py +0 -0
  148. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_cluster_util.py +0 -0
  149. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_clustering.py +0 -0
  150. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_config.py +0 -0
  151. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_dartsort.py +0 -0
  152. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_data_util.py +0 -0
  153. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_density.py +0 -0
  154. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_detect.py +0 -0
  155. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_drift_util.py +0 -0
  156. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_drifty_matching.py +0 -0
  157. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_enforce_decrease.py +0 -0
  158. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_grab_and_featurize.py +0 -0
  159. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_kmeans.py +0 -0
  160. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_linalg.py +0 -0
  161. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_localize_torch.py +0 -0
  162. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_matching.py +0 -0
  163. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_mixture_classifier.py +0 -0
  164. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_modeling.py +0 -0
  165. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_simkit.py +0 -0
  166. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_spiketorch.py +0 -0
  167. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_subtract.py +0 -0
  168. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_templates.py +0 -0
  169. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_threshold.py +0 -0
  170. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_torch_optimization_util.py +0 -0
  171. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_transform.py +0 -0
  172. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_util.py +0 -0
  173. {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_waveform_util.py +0 -0
  174. {dartsort-0.5.13 → dartsort-0.5.14}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.13
3
+ Version: 0.5.14
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -2683,7 +2683,9 @@ class TruncatedMixtureModel(BaseMixtureModel):
2683
2683
  # support of the likelihoods to drift away slightly from the weights',
2684
2684
  # and this can lead to slightly negative elbo changes. giving a little
2685
2685
  # more leeway than in em() here for when to panic about it.
2686
- assert (len(elbos) < 2) or (elbos[-1] > elbos[0] - 5e-2)
2686
+ # in fact, I'm going to comment this assertion out, but leaving the
2687
+ # comment here.
2688
+ # assert (len(elbos) < 2) or (elbos[-1] > elbos[0] - 5e-2)
2687
2689
 
2688
2690
  def score(
2689
2691
  self,
@@ -53,10 +53,10 @@ class DARTsortUserConfig:
53
53
  that there is enough data for clustering. See also subsampling_presence.
54
54
  Set to None to disable subsampling."""
55
55
 
56
- subsampling_presence: Annotated[float, Field(gt=0.0, le=1.0)] = 0.1
56
+ subsampling_presence: Annotated[float, Field(gt=0.0, le=1.0)] = 0.025
57
57
  """Early detection steps which have already found `subsampling_spikes`
58
58
  spikes are only allowed to end early if they additionally cover this
59
- fraction of the recording, to make sure there's good coverage of
59
+ fraction of the recording, to make sure there's some coverage of
60
60
  conditions for template estimation."""
61
61
 
62
62
  matching_iterations: int = 1
@@ -379,7 +379,7 @@ class DeveloperConfig(DARTsortUserConfig):
379
379
  # store extra intermediates@
380
380
  save_subtracted_waveforms: bool = False
381
381
  save_collisioncleaned_waveforms: bool = False
382
- always_save_final_tpca_feature: bool = False
382
+ always_save_detailed_features: bool = False
383
383
  precomputed_templates_npz: str | None = None
384
384
  save_everything_on_error: bool = False
385
385
  link_from: str | None = None
@@ -68,6 +68,7 @@ class DARTsortGroundTruthComparison:
68
68
  self._unsorted_detection = self._greedy_confusion = None
69
69
  if self.compute_unsorted_recall:
70
70
  self._calculate_greedy_confusion_and_detection()
71
+ self._matched_and_missed_cache = {}
71
72
  self._unit_info_cache = {}
72
73
 
73
74
  def _check(self):
@@ -338,6 +339,8 @@ class DARTsortGroundTruthComparison:
338
339
  assert self._unsorted_detection.shape == (gtns,)
339
340
 
340
341
  def matched_and_missed(self, gt_unit):
342
+ if gt_unit in self._matched_and_missed_cache:
343
+ return self._matched_and_missed_cache[gt_unit]
341
344
  (gt_spike_labels,) = self.comparison.get_labels1(gt_unit)
342
345
  assert self.gt_analysis.sorting.labels is not None
343
346
  in_gt_unit = self.gt_si_inds[
@@ -346,7 +349,9 @@ class DARTsortGroundTruthComparison:
346
349
  matched_gt_mask = gt_spike_labels == "TP"
347
350
  matched_gt_indices = in_gt_unit[matched_gt_mask]
348
351
  only_gt_indices = in_gt_unit[np.logical_not(matched_gt_mask)]
349
- return in_gt_unit, matched_gt_indices, only_gt_indices
352
+ ret = in_gt_unit, matched_gt_indices, only_gt_indices
353
+ self._matched_and_missed_cache[gt_unit] = ret
354
+ return ret
350
355
 
351
356
  def get_spikes_by_category(self, gt_unit, tested_unit=None):
352
357
  if tested_unit is None:
@@ -9,6 +9,7 @@ from pathlib import Path
9
9
  from sys import getrefcount
10
10
  from threading import Lock, local
11
11
  from typing import Any, Sequence, TypedDict
12
+ from warnings import catch_warnings, filterwarnings
12
13
 
13
14
  import h5py
14
15
  import numpy as np
@@ -539,8 +540,17 @@ class BasePeeler(BModule):
539
540
  channel_indices=None,
540
541
  margin=self.chunk_margin_samples,
541
542
  )
543
+ assert isinstance(chunk, np.ndarray)
542
544
  device = self.b.channel_index.device
543
- chunk = torch.tensor(chunk, device=device, dtype=self.dtype)
545
+ if device.type == "cpu":
546
+ chunk = torch.tensor(chunk, device=device, dtype=self.dtype)
547
+ else:
548
+ # we have to copy anyway to go to device, so working around the
549
+ # torch warning here. this way, can avoid blocking.
550
+ with catch_warnings():
551
+ filterwarnings("ignore", message="The given NumPy array is not ")
552
+ chunk = torch.from_numpy(chunk)
553
+ chunk = chunk.to(device=device, dtype=self.dtype, non_blocking=True)
544
554
  return chunk, chunk_end_samples, left_margin, right_margin
545
555
 
546
556
  def featurize_chunk_result(
@@ -20,6 +20,7 @@ from ..util.internal_config import (
20
20
  )
21
21
  from ..util.job_util import ensure_computation_config
22
22
  from ..util.spiketorch import grab_spikes, ptp, subtract_spikes_
23
+ from ..util.torch_util import torch_compile
23
24
  from ..util.waveform_util import get_relative_subset, make_channel_index
24
25
  from .peel_base import PeelingBatchResult
25
26
 
@@ -89,46 +90,32 @@ def check_residual_decrease(
89
90
  return None, {}
90
91
  assert orig_wfs is not None
91
92
 
92
- if local_whiteners is not None:
93
- assert channels is not None
94
- W = local_whiteners[channels]
95
-
96
- # remove nans
97
- if overwrite_orig_waveforms:
98
- orig_wfs = orig_wfs.nan_to_num_()
99
- else:
100
- orig_wfs = orig_wfs.nan_to_num()
101
- dn_wfs = dn_wfs.nan_to_num()
102
-
103
- # spatial mul -- putting temporal dim last here
104
- orig_wfs = W.bmm(orig_wfs.mT)
105
- dn_wfs = W.bmm(dn_wfs.mT)
106
-
107
- # temporal conv if needed
108
- if whitening_kernel is not None:
109
- *shp, t = orig_wfs.shape
110
- k = whitening_kernel[None, None]
111
- orig_wfs = F.conv1d(orig_wfs.view(-1, 1, t), k, padding="same")
112
- dn_wfs = F.conv1d(dn_wfs.view(-1, 1, t), k, padding="same")
113
- orig_wfs = orig_wfs.view(*shp, t)
114
- dn_wfs = dn_wfs.view(*shp, t)
93
+ orig_wfs = flatten_denan_and_whiten_batched(
94
+ orig_wfs,
95
+ channels,
96
+ local_whiteners,
97
+ whitening_kernel,
98
+ overwrite=overwrite_orig_waveforms,
99
+ )
100
+ dn_wfs = flatten_denan_and_whiten_batched(
101
+ dn_wfs, channels, local_whiteners, whitening_kernel
102
+ )
115
103
 
116
104
  if decrease_objective == "deconv":
117
105
  if overwrite_orig_waveforms:
118
- buf = orig_wfs.mul_(dn_wfs).nan_to_num_()
119
- conv = buf.sum(dim=(1, 2))
106
+ buf = orig_wfs.mul_(dn_wfs)
107
+ conv = buf.sum(dim=1)
120
108
  torch.square(dn_wfs, out=buf)
121
- norm = buf.nan_to_num_().sum(dim=(1, 2))
109
+ norm = buf.sum(dim=1)
122
110
  else:
123
- dn_wfs = dn_wfs.nan_to_num()
124
- conv = (orig_wfs * dn_wfs).nan_to_num_().sum(dim=(1, 2))
125
- norm = dn_wfs.square_().sum(dim=(1, 2))
111
+ conv = (orig_wfs * dn_wfs).sum(dim=1)
112
+ norm = dn_wfs.square_().sum(dim=1)
126
113
  reduction = conv.mul_(2.0).sub_(norm)
127
114
  threshold = threshold**2
128
115
  elif decrease_objective in ("norm", "normsq"):
129
- orig_decobj = orig_wfs.square().sum(dim=(1, 2))
130
- orig_wfs = orig_wfs.sub_(dn_wfs).nan_to_num_()
131
- new_decobj = orig_wfs.square_().sum(dim=(1, 2))
116
+ orig_decobj = orig_wfs.square().sum(dim=1)
117
+ orig_wfs = orig_wfs.sub_(dn_wfs)
118
+ new_decobj = orig_wfs.square_().sum(dim=1)
132
119
  if decrease_objective == "norm":
133
120
  orig_decobj = orig_decobj.sqrt_()
134
121
  new_decobj = new_decobj.sqrt_()
@@ -147,6 +134,51 @@ def check_residual_decrease(
147
134
  return keep, features
148
135
 
149
136
 
137
+ @torch_compile
138
+ def flatten_denan_and_whiten_batched(
139
+ wfs: torch.Tensor,
140
+ channels: torch.Tensor | None,
141
+ local_whiteners: Tensor | None,
142
+ whitening_kernel: Tensor | None,
143
+ overwrite: bool = False,
144
+ batch_size: int = 4096,
145
+ ):
146
+ N = wfs.shape[0]
147
+ T = wfs.shape[1]
148
+ C = wfs.shape[2]
149
+ if overwrite:
150
+ wfs = wfs.nan_to_num_()
151
+ else:
152
+ wfs = wfs.nan_to_num()
153
+
154
+ if local_whiteners is None:
155
+ return wfs.view(N, -1)
156
+
157
+ assert channels is not None
158
+
159
+ W = local_whiteners[channels]
160
+ wfs = W.bmm(wfs.mT)
161
+ if whitening_kernel is None:
162
+ return wfs.view(N, -1)
163
+
164
+ k = whitening_kernel[None, None]
165
+ NC = N * C
166
+ wfs = wfs.view(NC, 1, T)
167
+
168
+ for i0 in range(0, NC, batch_size):
169
+ i1 = min(NC, i0 + batch_size)
170
+ nb = i1 - i0
171
+ y = wfs[i0:i1]
172
+ if nb < batch_size:
173
+ y = F.pad(y, (0, 0, 0, 0, 0, batch_size - nb))
174
+ z = F.conv1d(y, k, padding="same")
175
+ if nb < batch_size:
176
+ z = z[:nb]
177
+ wfs[i0:i1] = z
178
+
179
+ return wfs.view(N, -1)
180
+
181
+
150
182
  ChunkSubtractionResult = namedtuple(
151
183
  "ChunkSubtractionResult",
152
184
  [
@@ -1,14 +1,15 @@
1
- from queue import Queue
2
- from threading import Thread
1
+ from queue import Empty, Queue
2
+ from threading import Lock, Thread, local
3
3
 
4
4
  import h5py
5
+ import numba
5
6
  import numpy as np
6
7
  import torch
7
8
  from torch.utils.data import DataLoader, Dataset, Sampler
8
9
 
9
10
  from ..util import nn_util, spikeio
10
11
  from ..util.logging_util import get_logger
11
- from ..util.spiketorch import get_relative_index, reindex
12
+ from ..util.spiketorch import get_relative_index, reindex, spawn_torch_rg
12
13
  from ..util.waveform_util import regularize_channel_index
13
14
  from .transform_base import BaseWaveformDenoiser
14
15
 
@@ -308,12 +309,17 @@ def get_noise_h5(
308
309
  spike_length_samples=121,
309
310
  rg: np.random.Generator,
310
311
  count: int | None = None,
312
+ local=True,
311
313
  ):
312
314
  if count is None:
313
315
  count = dset.shape[0]
314
316
 
315
- ii = rg.choice(count, size=len(channels), replace=False)
316
- ii.sort()
317
+ if local:
318
+ ii0 = rg.integers(count - len(channels)).item()
319
+ ii = slice(ii0, ii0 + len(channels))
320
+ else:
321
+ ii = rg.choice(count, size=len(channels), replace=False)
322
+ ii.sort()
317
323
 
318
324
  if dset.shape[1] > spike_length_samples:
319
325
  pad = dset.shape[1] - spike_length_samples
@@ -324,15 +330,24 @@ def get_noise_h5(
324
330
 
325
331
  # channels...
326
332
  cc = channel_index[channels]
327
- valid = cc < channel_index.shape[0]
328
- cc = np.where(valid, cc, 0)
329
333
  targ_shp = (cc.shape[0], noise_waveforms.shape[1], cc.shape[1])
330
- cc = np.broadcast_to(cc[:, None, :], targ_shp)
331
- noise_waveforms = np.take_along_axis(noise_waveforms, indices=cc, axis=2)
332
- # mask out
333
- noise_waveforms *= valid[:, None, :].astype(noise_waveforms.dtype)
334
+ out = np.empty_like(noise_waveforms, shape=targ_shp)
335
+ take_along_dim2_masked(noise_waveforms, out, cc, channel_index.shape[0])
334
336
 
335
- return torch.from_numpy(noise_waveforms)
337
+ return torch.from_numpy(out)
338
+
339
+
340
+ @numba.njit(parallel=True)
341
+ def take_along_dim2_masked(arr, out, channels, mask_val):
342
+ for i in numba.prange(arr.shape[0]): # type: ignore
343
+ wf = arr[i]
344
+ cc = channels[i]
345
+ for t in range(arr.shape[1]):
346
+ for j, c in enumerate(cc):
347
+ if c == mask_val:
348
+ out[i, t, j] = 0.0
349
+ else:
350
+ out[i, t, j] = wf[t, c]
336
351
 
337
352
 
338
353
  class NoneDataset(Dataset):
@@ -519,19 +534,29 @@ class AsyncBatchDataset(RefreshableDataset):
519
534
  n_examples,
520
535
  spike_length_samples=121,
521
536
  generator=None,
537
+ rg=None,
522
538
  chunk_size=2048,
523
539
  queue_chunks=8,
540
+ n_workers=1,
524
541
  ):
525
542
  super().__init__()
526
543
  self.spike_length_samples = spike_length_samples
527
544
  # note: although there's threading here, this is only used in one
528
545
  # thread, so that's okay.
529
- self.generator = generator
546
+ self._generator = generator
547
+ self._rg = rg
530
548
  self.n_examples = n_examples
549
+ self.n_workers = n_workers
550
+
551
+ self.lock = Lock()
552
+ self.locals = local()
553
+ self.locals.rg = None
554
+ self.locals.generator = None
531
555
 
532
556
  self.chunk_size = chunk_size
533
557
  self._queue = Queue(maxsize=queue_chunks)
534
- self._thread = None
558
+ self._batches = Queue()
559
+ self._threads = None
535
560
  self._indices = None
536
561
  self._cur_data_ix = None
537
562
  self._cur_chunk = None
@@ -543,7 +568,10 @@ class AsyncBatchDataset(RefreshableDataset):
543
568
 
544
569
  def destroy(self):
545
570
  self.bye = True
546
- if self._thread is not None:
571
+ if self._batches is not None:
572
+ if hasattr(self._batches, "shutdown"):
573
+ self._batches.shutdown(True) # type: ignore
574
+ if self._threads is not None:
547
575
  if hasattr(self._queue, "shutdown"):
548
576
  self._queue.shutdown(True) # type: ignore
549
577
  while True:
@@ -551,20 +579,26 @@ class AsyncBatchDataset(RefreshableDataset):
551
579
  self._queue.get()
552
580
  except Exception:
553
581
  break
554
- self._thread.join()
555
- assert not self._thread.is_alive()
556
- self._thread = None
582
+ for th in self._threads:
583
+ th.join()
584
+ for th in self._threads:
585
+ assert not th.is_alive()
586
+ self._threads = None
557
587
 
558
588
  def cleanup(self):
559
- if self._thread is not None:
560
- self._thread.join()
561
- assert not self._thread.is_alive()
562
- self._thread = None
589
+ if self._threads is not None:
590
+ for th in self._threads:
591
+ th.join()
592
+ for th in self._threads:
593
+ assert not th.is_alive()
594
+ self._threads = None
563
595
 
564
596
  def refresh(self, indices):
565
- assert self._thread is None, "Run cleanup()!"
597
+ assert self._threads is None, "Run cleanup()!"
566
598
  self._cur_data_ix = 0
567
599
  self._indices = indices
600
+ for batch in range(0, len(self._indices), self.chunk_size):
601
+ self._batches.put(batch)
568
602
  self._run_thread()
569
603
 
570
604
  def __getitem__(self, index):
@@ -572,8 +606,9 @@ class AsyncBatchDataset(RefreshableDataset):
572
606
  assert self._cur_data_ix is not None
573
607
 
574
608
  bs = len(index)
575
- my_indices = self._indices[self._cur_data_ix : self._cur_data_ix + bs]
576
- assert torch.equal(torch.asarray(index), my_indices)
609
+ # if self.n_workers == 1:
610
+ # my_indices = self._indices[self._cur_data_ix : self._cur_data_ix + bs]
611
+ # assert torch.equal(torch.asarray(index), my_indices)
577
612
 
578
613
  # need to batch up the chunks...
579
614
  if self._cur_chunk is None:
@@ -594,19 +629,45 @@ class AsyncBatchDataset(RefreshableDataset):
594
629
  self._cur_data_ix += bs
595
630
  return noise_batch
596
631
 
632
+ def _initialize_locals(self):
633
+ if self._rg is not None and not hasattr(self.locals, "rg"):
634
+ with self.lock:
635
+ self.locals.rg = self._rg.spawn(1)[0]
636
+ if self._generator is not None and not hasattr(self.locals, "generator"):
637
+ with self.lock:
638
+ self.locals.generator = spawn_torch_rg(self._generator)
639
+
640
+ @property
641
+ def rg(self):
642
+ return self.locals.rg
643
+
644
+ @property
645
+ def generator(self):
646
+ return self.locals.generator
647
+
597
648
  def _run_thread(self):
598
- self._thread = Thread(target=self._thread_main, daemon=True)
599
- self._thread.start()
649
+ self._threads = [
650
+ Thread(target=self._thread_main, daemon=True) for _ in range(self.n_workers)
651
+ ]
652
+ for th in self._threads:
653
+ th.start()
600
654
 
601
655
  def _thread_main(self):
602
656
  assert self._indices is not None
603
- for chunk_start in range(0, len(self._indices), self.chunk_size):
604
- chunk_end = min(len(self._indices), chunk_start + self.chunk_size)
657
+ assert self._batches is not None
658
+ self._initialize_locals()
659
+ while True:
660
+ try:
661
+ chunk_start = self._batches.get(timeout=0.01)
662
+
663
+ chunk_end = min(len(self._indices), chunk_start + self.chunk_size)
605
664
 
606
- index = self._indices[chunk_start:chunk_end]
607
- noise = self.load_batch(index)
665
+ index = self._indices[chunk_start:chunk_end]
666
+ noise = self.load_batch(index)
608
667
 
609
- self._queue.put(noise)
668
+ self._queue.put(noise)
669
+ except Empty:
670
+ break
610
671
  if self.bye:
611
672
  break
612
673
 
@@ -624,15 +685,19 @@ class AsyncSameChannelNoiseDataset(AsyncBatchDataset):
624
685
  channels,
625
686
  spike_length_samples=121,
626
687
  generator=None,
688
+ rg=None,
627
689
  chunk_size=2048,
628
690
  queue_chunks=8,
691
+ n_workers=1,
629
692
  ):
630
693
  super().__init__(
631
694
  n_examples=len(channels) if channels is not None else torch.inf,
632
695
  spike_length_samples=spike_length_samples,
633
696
  generator=generator,
697
+ rg=rg,
634
698
  chunk_size=chunk_size,
635
699
  queue_chunks=queue_chunks,
700
+ n_workers=n_workers,
636
701
  )
637
702
  self.channels = channels
638
703
  self.channel_index = channel_index
@@ -654,9 +719,11 @@ class AsyncSameChannelHDF5NoiseDataset(AsyncSameChannelNoiseDataset):
654
719
  generator=None,
655
720
  chunk_size=256,
656
721
  queue_chunks=8,
722
+ n_workers=1,
657
723
  dataset_name="residual",
658
724
  count_name="n_residuals",
659
725
  memmap=True,
726
+ memory=False,
660
727
  ):
661
728
  super().__init__(
662
729
  channels=channels,
@@ -664,9 +731,16 @@ class AsyncSameChannelHDF5NoiseDataset(AsyncSameChannelNoiseDataset):
664
731
  spike_length_samples=spike_length_samples,
665
732
  generator=generator,
666
733
  chunk_size=chunk_size,
734
+ rg=rg,
667
735
  queue_chunks=queue_chunks,
736
+ n_workers=n_workers,
668
737
  )
669
- if memmap:
738
+ if memory:
739
+ with h5py.File(hdf5_filename, "r", locking=False) as h5:
740
+ self.count = h5[count_name][()]
741
+ self.dset = h5[dataset_name][: self.count]
742
+ self.h5 = None
743
+ elif memmap:
670
744
  # this only works with contiguous datasets
671
745
  # hacky but h5 reading is so slow... and this is so fast...
672
746
  with h5py.File(hdf5_filename, "r", locking=False) as h5:
@@ -688,7 +762,6 @@ class AsyncSameChannelHDF5NoiseDataset(AsyncSameChannelNoiseDataset):
688
762
  )
689
763
  self.dset = self.h5[dataset_name]
690
764
  self.count = self.h5[count_name][()]
691
- self.rg = rg
692
765
  assert self.dset.shape[1] >= spike_length_samples
693
766
  assert self.dset.shape[2] == channel_index.shape[0]
694
767
 
@@ -719,6 +792,7 @@ class AsyncSameChannelRecordingNoiseDataset(AsyncSameChannelNoiseDataset):
719
792
  generator=None,
720
793
  chunk_size=2048,
721
794
  queue_chunks=8,
795
+ n_workers=1,
722
796
  ):
723
797
  super().__init__(
724
798
  channels=channels,
@@ -727,6 +801,7 @@ class AsyncSameChannelRecordingNoiseDataset(AsyncSameChannelNoiseDataset):
727
801
  generator=generator,
728
802
  chunk_size=chunk_size,
729
803
  queue_chunks=queue_chunks,
804
+ n_workers=n_workers,
730
805
  )
731
806
  self.recording = recording
732
807
 
@@ -72,6 +72,9 @@ class AmplitudeFeatures(BaseWaveformFeaturizer):
72
72
  ptp_max_amplitude or ptp_amplitude_vectors or log_peak_to_trough
73
73
  )
74
74
  self.compute_maxchan = ptp_max_amplitude or log_peak_to_trough
75
+ self.just_maxptp = ptp_max_amplitude and not (
76
+ log_peak_to_trough or ptp_amplitude_vectors or peak_amplitude_vectors
77
+ )
75
78
 
76
79
  super().__init__(name=names)
77
80
 
@@ -83,11 +86,16 @@ class AmplitudeFeatures(BaseWaveformFeaturizer):
83
86
  max_vectors = min_vectors = ptp_vectors = None
84
87
  maxs = mins = ptps = None
85
88
  if self.compute_minmax_vectors:
86
- max_vectors = waveforms.amax(dim=1)
87
- min_vectors = waveforms.amin(dim=1)
88
- ptp_vectors = max_vectors - min_vectors
89
- if self.compute_maxchan:
90
- assert ptp_vectors is not None
89
+ min_vectors, max_vectors = waveforms.aminmax(dim=1)
90
+ if self.just_maxptp:
91
+ assert min_vectors is not None
92
+ assert max_vectors is not None
93
+ ptp_vectors = max_vectors.sub_(min_vectors)
94
+ ptps = ptp_vectors.nan_to_num_().amax(dim=1)
95
+ elif self.compute_maxchan:
96
+ assert min_vectors is not None
97
+ assert max_vectors is not None
98
+ ptp_vectors = max_vectors.sub(min_vectors)
91
99
  maxchans = torch.argmax(ptp_vectors.nan_to_num(), dim=1, keepdim=True)
92
100
  if self.log_peak_to_trough:
93
101
  assert max_vectors is not None
@@ -98,7 +106,6 @@ class AmplitudeFeatures(BaseWaveformFeaturizer):
98
106
  ptps = maxs - mins
99
107
  elif self.ptp_max_amplitude:
100
108
  ptps = torch.take_along_dim(ptp_vectors, maxchans, dim=1)[:, 0]
101
-
102
109
  if self.ptp_amplitude_vectors:
103
110
  features[self.ptp_amplitude_vectors_name] = ptp_vectors
104
111
  if self.ptp_max_amplitude:
@@ -1,4 +1,5 @@
1
1
  import dataclasses
2
+ from math import isfinite
2
3
  from typing import cast
3
4
 
4
5
  import h5py
@@ -6,13 +7,10 @@ import numpy as np
6
7
  import pandas as pd
7
8
  import torch
8
9
  import torch.nn.functional as F
9
- from torch.utils.data import (
10
- DataLoader,
11
- StackDataset,
12
- TensorDataset,
13
- )
10
+ from torch.utils.data import DataLoader, StackDataset, TensorDataset
14
11
 
15
12
  from ..util.logging_util import get_logger, progrange
13
+ from ..util.multiprocessing_util import handle_negative_jobs
16
14
  from ..util.spiketorch import reindex, spawn_torch_rg
17
15
  from ._multichan_denoiser_kit import (
18
16
  AOTIndicesWeightedRandomBatchSampler,
@@ -43,6 +41,8 @@ class Decollider(BaseMultichannelDenoiser):
43
41
  name=None,
44
42
  name_prefix="",
45
43
  batch_size=512,
44
+ batches_per_chunk=4,
45
+ n_data_workers=1,
46
46
  learning_rate=2e-4,
47
47
  weight_decay=0.0,
48
48
  n_epochs=100,
@@ -127,6 +127,8 @@ class Decollider(BaseMultichannelDenoiser):
127
127
  warmup_lr=warmup_lr,
128
128
  )
129
129
  self.queue_chunks = queue_chunks
130
+ self.batches_per_chunk = batches_per_chunk
131
+ self.n_data_workers = n_data_workers
130
132
 
131
133
  self.inference_z_samples = inference_z_samples
132
134
  self.detach_amortizer = detach_amortizer
@@ -433,6 +435,8 @@ class Decollider(BaseMultichannelDenoiser):
433
435
  ):
434
436
  rg = np.random.default_rng(self.random_seed)
435
437
 
438
+ _, n_data_workers = handle_negative_jobs(self.n_data_workers)
439
+
436
440
  val_size = 0
437
441
  train_indices = slice(None)
438
442
  val_indices = None
@@ -463,7 +467,8 @@ class Decollider(BaseMultichannelDenoiser):
463
467
  rg=np.random.default_rng(rg.spawn(1)[0]),
464
468
  queue_chunks=self.queue_chunks,
465
469
  dataset_name=dataset_name,
466
- chunk_size=self.batch_size,
470
+ chunk_size=self.batches_per_chunk * self.batch_size,
471
+ n_workers=n_data_workers,
467
472
  )
468
473
  else:
469
474
  logger.dartsortdebug("Load Decollider train noise data from recording.")
@@ -474,6 +479,7 @@ class Decollider(BaseMultichannelDenoiser):
474
479
  spike_length_samples=spike_length_samples,
475
480
  generator=spawn_torch_rg(rg),
476
481
  queue_chunks=self.queue_chunks,
482
+ n_workers=n_data_workers,
477
483
  )
478
484
  if self.cycle_loss_alpha and can_load_h5:
479
485
  logger.dartsortdebug(
@@ -488,6 +494,7 @@ class Decollider(BaseMultichannelDenoiser):
488
494
  queue_chunks=self.queue_chunks,
489
495
  dataset_name=dataset_name,
490
496
  chunk_size=self.batch_size,
497
+ n_workers=n_data_workers,
491
498
  )
492
499
  elif self.cycle_loss_alpha:
493
500
  logger.dartsortdebug("Load Decollider cycle noise data from recording.")
@@ -498,6 +505,7 @@ class Decollider(BaseMultichannelDenoiser):
498
505
  spike_length_samples=spike_length_samples,
499
506
  generator=spawn_torch_rg(rg),
500
507
  queue_chunks=self.queue_chunks,
508
+ n_workers=n_data_workers,
501
509
  )
502
510
  else:
503
511
  train_cycle_noise_dataset = NoneDataset(len(train_channels))
@@ -556,8 +564,8 @@ class Decollider(BaseMultichannelDenoiser):
556
564
  # val set does not need shuffling
557
565
  val_loader = DataLoader(
558
566
  val_dataset,
559
- num_workers=self.n_data_workers, # type: ignore # ty: ignore[x]
560
- persistent_workers=bool(self.n_data_workers),
567
+ num_workers=n_data_workers, # type: ignore # ty: ignore[x]
568
+ persistent_workers=bool(n_data_workers),
561
569
  batch_size=self.batch_size,
562
570
  )
563
571
  val_data = DecolliderDataLoader(
@@ -652,6 +660,8 @@ class Decollider(BaseMultichannelDenoiser):
652
660
  train_losses = {
653
661
  k: v.item() / len(train_data) for k, v in train_losses.items()
654
662
  }
663
+ if not all(isfinite(v) for v in train_losses.values()):
664
+ raise ValueError(f"Denoiser training diverged: {train_losses}.")
655
665
  train_records.append({**train_losses})
656
666
 
657
667
  # Validation phase (only if val_loader is not None)
@@ -616,13 +616,19 @@ def _add_localization_and_ampvec(fc):
616
616
  more = []
617
617
 
618
618
  do_ptp_amp = do_feats and fc.save_amplitudes
619
- do_peak_vec = do_feats and (
620
- fc.do_localization
621
- and fc.localization_amplitude_type == "peak"
622
- and not fc.nn_localization
619
+ do_peak_vec = (
620
+ do_feats
621
+ and fc.do_localization
622
+ and (fc.localization_amplitude_type == "peak")
623
+ and (fc.save_amplitude_vectors or not fc.nn_localization)
624
+ )
625
+ do_ptp_vec = (
626
+ do_feats
627
+ and fc.do_localization
628
+ and (fc.localization_amplitude_type == "ptp")
629
+ and (fc.save_amplitude_vectors or not fc.nn_localization)
623
630
  )
624
- do_ptp_vec = do_feats and fc.save_amplitudes
625
- do_logptt = do_feats and fc.save_amplitudes
631
+ do_logptt = do_feats and fc.save_all_amplitudes
626
632
  do_any_amp = do_peak_vec or do_ptp_vec or do_ptp_amp or do_logptt
627
633
  if do_any_amp or (do_feats and fc.save_all_amplitudes):
628
634
  more.append(