DARTsort 0.5.12__tar.gz → 0.5.14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. {dartsort-0.5.12 → dartsort-0.5.14}/PKG-INFO +1 -1
  2. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/agglomerate.py +2 -7
  3. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/clustering.py +22 -3
  4. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/clustering_features.py +70 -22
  5. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/mixture.py +4 -2
  6. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/config.py +3 -3
  7. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/analysis.py +8 -1
  8. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/comparison.py +8 -3
  9. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/grab.py +2 -0
  10. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching.py +13 -1
  11. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/peel_base.py +13 -1
  12. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/peel_lib.py +64 -32
  13. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/_multichan_denoiser_kit.py +109 -34
  14. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/amortized_localization.py +3 -3
  15. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/amplitudes.py +13 -6
  16. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/decollider.py +18 -8
  17. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/mixture_classifier.py +102 -72
  18. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/pipeline.py +12 -6
  19. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/temporal_pca.py +15 -3
  20. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/data_util.py +25 -4
  21. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/internal_config.py +7 -5
  22. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/main_util.py +2 -1
  23. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/noise_util.py +18 -2
  24. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/peel_util.py +16 -0
  25. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/analysis_plots.py +99 -6
  26. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/mixture.py +28 -7
  27. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/scatterplots.py +1 -1
  28. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/vismain.py +2 -0
  29. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/PKG-INFO +1 -1
  30. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/SOURCES.txt +1 -52
  31. dartsort-0.5.14/tests/test_agglomerate.py +35 -0
  32. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_data_util.py +1 -0
  33. dartsort-0.5.14/tests/test_mixture_classifier.py +138 -0
  34. dartsort-0.5.12/notebook/basic_integration_test.py +0 -137
  35. dartsort-0.5.12/notebook/ci diag.py +0 -573
  36. dartsort-0.5.12/notebook/dartsort_dredge_demo.ipynb +0 -437
  37. dartsort-0.5.12/notebook/dartsort_dredge_demo.py +0 -144
  38. dartsort-0.5.12/notebook/data test.py +0 -259
  39. dartsort-0.5.12/notebook/dead reckoning.py +0 -361
  40. dartsort-0.5.12/notebook/denoise np1.py +0 -356
  41. dartsort-0.5.12/notebook/denoise np2.py +0 -480
  42. dartsort-0.5.12/notebook/exploratory.py +0 -431
  43. dartsort-0.5.12/notebook/geom debug.py +0 -167
  44. dartsort-0.5.12/notebook/hdbscan_clustering_5min.py +0 -1294
  45. dartsort-0.5.12/notebook/hybrid 10.py +0 -1662
  46. dartsort-0.5.12/notebook/hybrid cluster.py +0 -475
  47. dartsort-0.5.12/notebook/hybrid_11.py +0 -1214
  48. dartsort-0.5.12/notebook/initial training.py +0 -162
  49. dartsort-0.5.12/notebook/make hybrid.py +0 -1376
  50. dartsort-0.5.12/notebook/mgh.py +0 -514
  51. dartsort-0.5.12/notebook/np1 feats.py +0 -1211
  52. dartsort-0.5.12/notebook/np2 processing version c.py +0 -321
  53. dartsort-0.5.12/notebook/paperfigs.py +0 -907
  54. dartsort-0.5.12/notebook/pdists.py +0 -641
  55. dartsort-0.5.12/notebook/postcheck.py +0 -803
  56. dartsort-0.5.12/notebook/ptp vae.py +0 -206
  57. dartsort-0.5.12/notebook/re clust.py +0 -893
  58. dartsort-0.5.12/notebook/reg fig loop.py +0 -723
  59. dartsort-0.5.12/notebook/reg_sim_exp.py +0 -493
  60. dartsort-0.5.12/notebook/relocating.py +0 -284
  61. dartsort-0.5.12/notebook/relocating_all.py +0 -182
  62. dartsort-0.5.12/notebook/restats +0 -0
  63. dartsort-0.5.12/notebook/sortStinger1.py +0 -1590
  64. dartsort-0.5.12/notebook/sortStinger3.py +0 -1541
  65. dartsort-0.5.12/notebook/sortStinger4.py +0 -1835
  66. dartsort-0.5.12/notebook/sortStinger5.py +0 -1586
  67. dartsort-0.5.12/notebook/sortStinger6.py +0 -1915
  68. dartsort-0.5.12/notebook/spike farm.py +0 -100
  69. dartsort-0.5.12/notebook/ss_ensemble_sim.py +0 -1965
  70. dartsort-0.5.12/notebook/ss_ensemble_sim_drift.py +0 -1928
  71. dartsort-0.5.12/notebook/ss_sim.py +0 -1965
  72. dartsort-0.5.12/notebook/stan diagnostics.py +0 -207
  73. dartsort-0.5.12/notebook/storytelling c.py +0 -686
  74. dartsort-0.5.12/notebook/storytelling.py +0 -684
  75. dartsort-0.5.12/notebook/subtract_with_phaseshift_denoiser.py +0 -260
  76. dartsort-0.5.12/notebook/tests/debug_matching.py +0 -454
  77. dartsort-0.5.12/notebook/tests/test_agreement.py +0 -367
  78. dartsort-0.5.12/notebook/tests/test_amplitude_scaling.py +0 -737
  79. dartsort-0.5.12/notebook/tests/test_batch_optim.py +0 -679
  80. dartsort-0.5.12/notebook/tests/test_ibme.py +0 -86
  81. dartsort-0.5.12/notebook/tests/test_mm_initialization.py +0 -142
  82. dartsort-0.5.12/notebook/tests/test_normxcorr1d.py +0 -63
  83. dartsort-0.5.12/notebook/tests/test_reassignment.py +0 -422
  84. dartsort-0.5.12/notebook/tests/test_relocation.py +0 -230
  85. dartsort-0.5.12/notebook/thomas.py +0 -1004
  86. dartsort-0.5.12/tests/test_mixture_classifier.py +0 -65
  87. {dartsort-0.5.12 → dartsort-0.5.14}/.github/workflows/ci.yml +0 -0
  88. {dartsort-0.5.12 → dartsort-0.5.14}/.github/workflows/ci_windows.yml +0 -0
  89. {dartsort-0.5.12 → dartsort-0.5.14}/.github/workflows/deploy.yml +0 -0
  90. {dartsort-0.5.12 → dartsort-0.5.14}/.github/workflows/docs.yml +0 -0
  91. {dartsort-0.5.12 → dartsort-0.5.14}/.gitignore +0 -0
  92. {dartsort-0.5.12 → dartsort-0.5.14}/README.md +0 -0
  93. {dartsort-0.5.12 → dartsort-0.5.14}/docs/index.md +0 -0
  94. {dartsort-0.5.12 → dartsort-0.5.14}/docs/main_api.md +0 -0
  95. {dartsort-0.5.12 → dartsort-0.5.14}/docs/toolkit_api.md +0 -0
  96. {dartsort-0.5.12 → dartsort-0.5.14}/environment.yml +0 -0
  97. {dartsort-0.5.12 → dartsort-0.5.14}/notebook/.keep +0 -0
  98. {dartsort-0.5.12 → dartsort-0.5.14}/pyproject.toml +0 -0
  99. {dartsort-0.5.12 → dartsort-0.5.14}/setup.cfg +0 -0
  100. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/__init__.py +0 -0
  101. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/cli.py +0 -0
  102. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/__init__.py +0 -0
  103. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/cluster_util.py +0 -0
  104. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/density.py +0 -0
  105. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/forward_backward.py +0 -0
  106. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/kmeans.py +0 -0
  107. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/merge.py +0 -0
  108. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/modes.py +0 -0
  109. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/refine_util.py +0 -0
  110. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/detect/__init__.py +0 -0
  111. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/detect/detect.py +0 -0
  112. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/detect/detect_filters.py +0 -0
  113. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/__init__.py +0 -0
  114. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/hybrid_util.py +0 -0
  115. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  116. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  117. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/simkit.py +0 -0
  118. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/simlib.py +0 -0
  119. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/localize/__init__.py +0 -0
  120. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/localize/localize_torch.py +0 -0
  121. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/localize/localize_util.py +0 -0
  122. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/main.py +0 -0
  123. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/__init__.py +0 -0
  124. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/__init__.py +0 -0
  125. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  126. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/drifty.py +0 -0
  127. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  128. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  129. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  130. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/reduction_template.py +0 -0
  131. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/shaver.py +0 -0
  132. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/subtract.py +0 -0
  133. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/threshold.py +0 -0
  134. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  135. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/detect_np1.pt +0 -0
  136. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/detect_np2.pt +0 -0
  137. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  138. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  139. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  140. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  141. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  142. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  143. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/__init__.py +0 -0
  144. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/get_templates.py +0 -0
  145. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/postprocess_util.py +0 -0
  146. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/realignment.py +0 -0
  147. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/template_util.py +0 -0
  148. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/templates.py +0 -0
  149. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/templib.py +0 -0
  150. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/__init__.py +0 -0
  151. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/all_transformers.py +0 -0
  152. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/denoising_scorer.py +0 -0
  153. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/enforce_decrease.py +0 -0
  154. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/fixed_prop.py +0 -0
  155. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/interp.py +0 -0
  156. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/localize.py +0 -0
  157. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/matching_denoiser.py +0 -0
  158. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/reduction.py +0 -0
  159. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  160. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/supervised_denoiser.py +0 -0
  161. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/transform_base.py +2 -2
  162. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/whiten.py +0 -0
  163. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/__init__.py +0 -0
  164. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/cli_util.py +0 -0
  165. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/drift_util.py +0 -0
  166. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/interpolation_util.py +0 -0
  167. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/job_util.py +0 -0
  168. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/logging_util.py +0 -0
  169. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/more_operators.py +0 -0
  170. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/motion.py +0 -0
  171. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/multiprocessing_util.py +0 -0
  172. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/nn_util.py +0 -0
  173. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/preprocess_util.py +0 -0
  174. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/py_util.py +0 -0
  175. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/registration_util.py +0 -0
  176. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/residual_util.py +0 -0
  177. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/sparse_util.py +0 -0
  178. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/spikeio.py +0 -0
  179. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/spiketorch.py +0 -0
  180. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/testing_util/__init__.py +0 -0
  181. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  182. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  183. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/torch_optimization_util.py +0 -0
  184. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/torch_util.py +0 -0
  185. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/waveform_util.py +0 -0
  186. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/__init__.py +0 -0
  187. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/colors.py +0 -0
  188. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/gt.py +0 -0
  189. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/layout.py +0 -0
  190. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/metrics_util.py +0 -0
  191. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/recanim.py +0 -0
  192. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/sorting.py +0 -0
  193. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/unit.py +0 -0
  194. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/unit_comparison.py +0 -0
  195. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/versus.py +0 -0
  196. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/waveforms.py +0 -0
  197. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/dependency_links.txt +0 -0
  198. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/entry_points.txt +0 -0
  199. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/requires.txt +0 -0
  200. {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/top_level.txt +0 -0
  201. {dartsort-0.5.12 → dartsort-0.5.14}/tests/conftest.py +0 -0
  202. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_alignment.py +0 -0
  203. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_cluster_util.py +0 -0
  204. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_clustering.py +0 -0
  205. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_config.py +0 -0
  206. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_dartsort.py +0 -0
  207. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_density.py +0 -0
  208. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_detect.py +0 -0
  209. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_drift_util.py +0 -0
  210. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_drifty_matching.py +0 -0
  211. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_enforce_decrease.py +0 -0
  212. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_grab_and_featurize.py +0 -0
  213. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_kmeans.py +0 -0
  214. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_linalg.py +0 -0
  215. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_localize_torch.py +0 -0
  216. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_matching.py +0 -0
  217. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_modeling.py +0 -0
  218. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_simkit.py +0 -0
  219. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_spiketorch.py +0 -0
  220. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_subtract.py +0 -0
  221. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_templates.py +0 -0
  222. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_threshold.py +0 -0
  223. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_torch_optimization_util.py +0 -0
  224. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_transform.py +0 -0
  225. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_util.py +0 -0
  226. {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_waveform_util.py +0 -0
  227. {dartsort-0.5.12 → dartsort-0.5.14}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.12
3
+ Version: 0.5.14
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -770,7 +770,7 @@ def combine_gmm_scores(
770
770
  **{
771
771
  f"{new_prefix}_candidates": candidates,
772
772
  f"{new_prefix}_responsibilities": responsibilities,
773
- f"{new_prefix}_logliks": logliks,
773
+ f"{new_prefix}_log_liks": logliks,
774
774
  }
775
775
  )
776
776
 
@@ -826,7 +826,7 @@ def combine_gmm_scores(
826
826
  **{
827
827
  f"{new_prefix}_candidates": cand,
828
828
  f"{new_prefix}_responsibilities": mergedr,
829
- f"{new_prefix}_logliks": mergedl,
829
+ f"{new_prefix}_log_liks": mergedl,
830
830
  },
831
831
  )
832
832
 
@@ -873,11 +873,6 @@ def deduplicate_spikes(
873
873
  radius_ms: float = -1.0,
874
874
  score_by=("merged_log_liks", "gmm_log_liks", "scores"),
875
875
  ) -> DARTsortSorting:
876
- """The lower-scoring of any spikes within radius_samples of each other is relabeled to -1.
877
-
878
- scores are grabbed from the first entry of score_by which is a valid property of the sorting.
879
- If it's mu
880
- """
881
876
  if radius_ms < 0 or sorting.labels is None:
882
877
  return sorting
883
878
 
@@ -155,10 +155,23 @@ class Clusterer:
155
155
  def handle_sampling(
156
156
  self, features: SimpleMatrixFeatures
157
157
  ) -> tuple[Literal[False], slice] | tuple[Literal[True], np.ndarray]:
158
+ if features.keep_mask is not None:
159
+ mask_ix = np.flatnonzero(features.keep_mask)
160
+ else:
161
+ mask_ix = None
158
162
  if self.sampling_cfg is None:
159
- return False, slice(None)
163
+ if mask_ix is not None:
164
+ return True, mask_ix
165
+ else:
166
+ return False, slice(None)
160
167
  elif features.n <= self.sampling_cfg.n_waveforms_fit:
161
- return False, slice(None)
168
+ if mask_ix is not None:
169
+ return True, mask_ix
170
+ else:
171
+ return False, slice(None)
172
+
173
+ if mask_ix is not None:
174
+ features = features.mask(mask_ix)
162
175
 
163
176
  weights = fit_reweighting(
164
177
  voltages=features.signed_amplitudes,
@@ -173,6 +186,8 @@ class Clusterer:
173
186
  replace=False,
174
187
  )
175
188
  ixs.sort()
189
+ if mask_ix is not None:
190
+ ixs = mask_ix[ixs]
176
191
  return True, ixs
177
192
 
178
193
  def cluster(
@@ -432,16 +447,20 @@ class DensityPeaksClusterer(Clusterer):
432
447
  kdtree = res["kdtree"]
433
448
  assert isinstance(ixs, np.ndarray)
434
449
  rest = np.setdiff1d(np.arange(len(X)), ixs)
450
+ Xr = X[rest]
451
+ isna = np.isnan(Xr).any(axis=1)
452
+ np.nan_to_num(Xr, copy=False)
435
453
  other_labels = density.nearest_neighbor_assign(
436
454
  kdtree,
437
455
  res["labels"],
438
- X[rest],
456
+ Xr,
439
457
  radius_search=self.radius_search,
440
458
  workers=self.workers,
441
459
  )
442
460
  labels = cluster_util.combine_disjoint(
443
461
  ixs, res["labels"], rest, other_labels
444
462
  )
463
+ labels[rest[isna]] = -1
445
464
  else:
446
465
  kdtree = None
447
466
  labels = res["labels"]
@@ -1,4 +1,5 @@
1
1
  from typing import Self, cast
2
+ from logging import WARN
2
3
 
3
4
  import h5py
4
5
  import numpy as np
@@ -38,9 +39,14 @@ class SimpleMatrixFeatures:
38
39
  xyza: np.ndarray | None
39
40
  signed_amplitudes: np.ndarray
40
41
  amplitudes: np.ndarray
42
+ keep_mask: np.ndarray | None
41
43
 
42
44
  def mask(self, mask) -> Self:
43
45
  a = self.amplitudes[mask]
46
+ if self.keep_mask is not None:
47
+ k = self.keep_mask[mask]
48
+ else:
49
+ k = None
44
50
  return self.__class__(
45
51
  n=a.shape[0],
46
52
  features=self.features[mask],
@@ -50,6 +56,7 @@ class SimpleMatrixFeatures:
50
56
  xyza=None if self.xyza is None else self.xyza[mask],
51
57
  signed_amplitudes=self.signed_amplitudes[mask],
52
58
  amplitudes=a,
59
+ keep_mask=k,
53
60
  )
54
61
 
55
62
  @classmethod
@@ -68,14 +75,19 @@ class SimpleMatrixFeatures:
68
75
  )
69
76
  if xyza is not None:
70
77
  x = xyza[:, 0]
71
- if not _allfinite(x):
72
- raise ValueError(_numbers_error_str("x", x))
78
+ _check_numbers(
79
+ "x", x, raise_for_numerics=clustering_features_cfg.raise_for_numerics
80
+ )
73
81
  z = xyza[:, 2]
74
- if not _allfinite(z):
75
- raise ValueError(_numbers_error_str("z", z))
82
+ _check_numbers(
83
+ "z", z, raise_for_numerics=clustering_features_cfg.raise_for_numerics
84
+ )
76
85
  z_reg = motion.correct_s(t_s, z)
77
- if not _allfinite(z_reg):
78
- raise ValueError(_numbers_error_str("z_reg", z_reg))
86
+ _check_numbers(
87
+ "z_reg",
88
+ z_reg,
89
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
90
+ )
79
91
  else:
80
92
  x = z = z_reg = None
81
93
 
@@ -96,14 +108,20 @@ class SimpleMatrixFeatures:
96
108
  amp = getattr(sorting, clustering_features_cfg.amplitudes_dataset_name)
97
109
  if clustering_features_cfg.use_amplitude:
98
110
  assert amp is not None
99
- if not _allfinite(amp):
100
- raise ValueError(_numbers_error_str("amp", amp))
111
+ _check_numbers(
112
+ "amp",
113
+ amp,
114
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
115
+ )
101
116
  ampft = amp.copy()
102
117
  if clustering_features_cfg.log_transform_amplitude:
103
118
  ampft = np.log(clustering_features_cfg.amp_log_c + ampft)
104
119
  ampft *= clustering_features_cfg.amp_scale
105
- if not _allfinite(ampft):
106
- raise ValueError(_numbers_error_str("ampft", ampft))
120
+ _check_numbers(
121
+ "ampft",
122
+ ampft,
123
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
124
+ )
107
125
  features.append(ampft[:, None])
108
126
 
109
127
  v = getattr(sorting, clustering_features_cfg.voltages_dataset_name, None)
@@ -124,8 +142,11 @@ class SimpleMatrixFeatures:
124
142
  rank=clustering_features_cfg.n_main_channel_pcs,
125
143
  dataset_name=clustering_features_cfg.pca_dataset_name,
126
144
  )
127
- if not _allfinite(pcs):
128
- raise ValueError(_numbers_error_str("No motion pcs", pcs))
145
+ _check_numbers(
146
+ "No motion pcs",
147
+ pcs,
148
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
149
+ )
129
150
  elif do_pcs and clustering_features_cfg.motion_aware:
130
151
  shifts, n_pitches_shift = motion.pitch_shifts(
131
152
  sorting=sorting,
@@ -144,8 +165,11 @@ class SimpleMatrixFeatures:
144
165
  mask = np.broadcast_to(mask, len(schan))
145
166
  if hasattr(sorting, clustering_features_cfg.pca_dataset_name):
146
167
  pcs = getattr(sorting, clustering_features_cfg.pca_dataset_name)
147
- if not _allfinite(pcs):
148
- raise ValueError(_numbers_error_str("sorting pcs", pcs))
168
+ _check_numbers(
169
+ "sorting pcs",
170
+ pcs,
171
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
172
+ )
149
173
  erp, pcs = interpolate_by_chunk(
150
174
  mask=mask,
151
175
  dataset=pcs,
@@ -158,8 +182,11 @@ class SimpleMatrixFeatures:
158
182
  params=clustering_features_cfg.interp_params,
159
183
  )
160
184
  pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
161
- if not _allfinite(pcs):
162
- raise ValueError(_numbers_error_str("sorting interp pcs", pcs))
185
+ _check_numbers(
186
+ "sorting interp pcs",
187
+ pcs,
188
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
189
+ )
163
190
  else:
164
191
  assert sorting.parent_h5_path is not None
165
192
  with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
@@ -175,8 +202,11 @@ class SimpleMatrixFeatures:
175
202
  params=clustering_features_cfg.interp_params,
176
203
  )
177
204
  pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
178
- if not _allfinite(pcs):
179
- raise ValueError(_numbers_error_str("h5 interp pcs", pcs))
205
+ _check_numbers(
206
+ "h5 interp pcs",
207
+ pcs,
208
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
209
+ )
180
210
 
181
211
  if do_pcs:
182
212
  assert pcs is not None
@@ -192,8 +222,11 @@ class SimpleMatrixFeatures:
192
222
  else:
193
223
  assert pctf in ("none", None)
194
224
  pcs *= clustering_features_cfg.pc_scale
195
- if not _allfinite(pcs):
196
- raise ValueError(_numbers_error_str(f"{pctf} pcs", pcs))
225
+ _check_numbers(
226
+ f"{pctf} pcs",
227
+ pcs,
228
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
229
+ )
197
230
  if torch.is_tensor(pcs):
198
231
  pcs = pcs.numpy(force=True)
199
232
  features.append(pcs)
@@ -203,6 +236,9 @@ class SimpleMatrixFeatures:
203
236
  features = np.concatenate(features, axis=1)
204
237
  else:
205
238
  features = np.empty((n, 0))
239
+ keep = np.atleast_1d(np.isfinite(features).all(axis=1))
240
+ if keep.all():
241
+ keep = None
206
242
  return cls(
207
243
  n=n,
208
244
  features=features,
@@ -212,6 +248,7 @@ class SimpleMatrixFeatures:
212
248
  xyza=xyza,
213
249
  amplitudes=amp,
214
250
  signed_amplitudes=samp,
251
+ keep_mask=keep,
215
252
  )
216
253
 
217
254
 
@@ -331,7 +368,18 @@ def _allfinite(x):
331
368
  return np.isfinite(x).all()
332
369
 
333
370
 
334
- def _numbers_error_str(name: str, x: np.ndarray):
371
+ def _check_numbers(name: str, x: np.ndarray, raise_for_numerics=False):
335
372
  if isinstance(x, torch.Tensor):
336
373
  x = x.numpy(force=True)
337
- return f"{name}: {np.isposinf(x).sum()} +inf, {np.isneginf(x).sum()} -inf, {np.isnan(x).sum()} nan."
374
+
375
+ npinf = np.isposinf(x).sum()
376
+ nninf = np.isneginf(x).sum()
377
+ nna = np.isnan(x).sum()
378
+ if not (npinf + nninf + nna):
379
+ return
380
+
381
+ err = f"{name}: {npinf} +inf, {nninf} -inf, {nna} nan."
382
+ if raise_for_numerics:
383
+ raise ValueError(err)
384
+ else:
385
+ logger.warn(err)
@@ -2683,7 +2683,9 @@ class TruncatedMixtureModel(BaseMixtureModel):
2683
2683
  # support of the likelihoods to drift away slightly from the weights',
2684
2684
  # and this can lead to slightly negative elbo changes. giving a little
2685
2685
  # more leeway than in em() here for when to panic about it.
2686
- assert (len(elbos) < 2) or (elbos[-1] > elbos[0] - 5e-2)
2686
+ # in fact, I'm going to comment this assertion out, but leaving the
2687
+ # comment here.
2688
+ # assert (len(elbos) < 2) or (elbos[-1] > elbos[0] - 5e-2)
2687
2689
 
2688
2690
  def score(
2689
2691
  self,
@@ -4381,7 +4383,7 @@ def get_full_neighborhood_data(
4381
4383
  assert xfull.shape[2] == full_neighborhoods.b.neighborhoods.shape[1]
4382
4384
  frank = xfull.shape[1]
4383
4385
  if clustering_features_cfg is not None:
4384
- assert frank == clustering_features_cfg.feature_rank
4386
+ assert 0 < frank <= clustering_features_cfg.feature_rank
4385
4387
  assert xfull.shape[0] == full_neighborhoods.b.neighborhood_ids.shape[0]
4386
4388
  xfull = xfull.view(len(xfull), -1)
4387
4389
  xfull = xfull.nan_to_num_()
@@ -53,10 +53,10 @@ class DARTsortUserConfig:
53
53
  that there is enough data for clustering. See also subsampling_presence.
54
54
  Set to None to disable subsampling."""
55
55
 
56
- subsampling_presence: Annotated[float, Field(gt=0.0, le=1.0)] = 0.1
56
+ subsampling_presence: Annotated[float, Field(gt=0.0, le=1.0)] = 0.025
57
57
  """Early detection steps which have already found `subsampling_spikes`
58
58
  spikes are only allowed to end early if they additionally cover this
59
- fraction of the recording, to make sure there's good coverage of
59
+ fraction of the recording, to make sure there's some coverage of
60
60
  conditions for template estimation."""
61
61
 
62
62
  matching_iterations: int = 1
@@ -379,7 +379,7 @@ class DeveloperConfig(DARTsortUserConfig):
379
379
  # store extra intermediates@
380
380
  save_subtracted_waveforms: bool = False
381
381
  save_collisioncleaned_waveforms: bool = False
382
- always_save_final_tpca_feature: bool = False
382
+ always_save_detailed_features: bool = False
383
383
  precomputed_templates_npz: str | None = None
384
384
  save_everything_on_error: bool = False
385
385
  link_from: str | None = None
@@ -108,6 +108,8 @@ class DARTsortAnalysis:
108
108
  "localizations",
109
109
  "positions",
110
110
  ),
111
+ compute_extensions=("correlograms",),
112
+ estimate_si_sparsity=False,
111
113
  try_backup_amplitudes=("amplitudes", "ptp_amplitudes"),
112
114
  quality_metrics=(
113
115
  "sliding_rp_violation",
@@ -288,6 +290,8 @@ class DARTsortAnalysis:
288
290
  motion=motion,
289
291
  features_cfg=clustering_features_cfg,
290
292
  compute_extensions_if_templates=None,
293
+ compute_extensions=compute_extensions,
294
+ estimate_si_sparsity=estimate_si_sparsity,
291
295
  )
292
296
  analyzer.compute("quality_metrics", metric_names=quality_metrics)
293
297
  qc_df = analyzer.get_extension("quality_metrics").get_data()
@@ -379,7 +383,10 @@ class DARTsortAnalysis:
379
383
  return self._summary_df
380
384
  df = self.quality_df.copy()
381
385
  uids = self.sorting.unit_ids
382
- df["ds_unit_amplitudes"] = pd.Series(index=uids, data=self.unit_amplitudes())
386
+ if self.template_data is not None:
387
+ df["ds_unit_amplitudes"] = pd.Series(
388
+ index=uids, data=self.unit_amplitudes()
389
+ )
383
390
  df["ds_firing_rates"] = pd.Series(index=uids, data=self.firing_rates())
384
391
  self._summary_df = df
385
392
  return df
@@ -68,6 +68,7 @@ class DARTsortGroundTruthComparison:
68
68
  self._unsorted_detection = self._greedy_confusion = None
69
69
  if self.compute_unsorted_recall:
70
70
  self._calculate_greedy_confusion_and_detection()
71
+ self._matched_and_missed_cache = {}
71
72
  self._unit_info_cache = {}
72
73
 
73
74
  def _check(self):
@@ -135,12 +136,12 @@ class DARTsortGroundTruthComparison:
135
136
  return res
136
137
 
137
138
  def _unit_info_dataframe(self, force_distances=False, perf_only=False):
138
- amplitudes = self.gt_analysis.unit_amplitudes()
139
139
  firing_rates = self.gt_analysis.firing_rates()
140
140
  df = self.comparison.get_performance()
141
141
  assert isinstance(df, pd.DataFrame)
142
142
  df = df.astype(float) # not sure what the problem was...
143
- df["gt_ptp_amplitude"] = amplitudes
143
+ if self.gt_analysis.template_data is not None:
144
+ df["gt_ptp_amplitude"] = self.gt_analysis.unit_amplitudes()
144
145
  df["gt_firing_rate"] = firing_rates
145
146
  df["gt_sorter_name"] = self.gt_analysis.name or ""
146
147
  df["tested_sorter_name"] = self.tested_analysis.name or ""
@@ -338,6 +339,8 @@ class DARTsortGroundTruthComparison:
338
339
  assert self._unsorted_detection.shape == (gtns,)
339
340
 
340
341
  def matched_and_missed(self, gt_unit):
342
+ if gt_unit in self._matched_and_missed_cache:
343
+ return self._matched_and_missed_cache[gt_unit]
341
344
  (gt_spike_labels,) = self.comparison.get_labels1(gt_unit)
342
345
  assert self.gt_analysis.sorting.labels is not None
343
346
  in_gt_unit = self.gt_si_inds[
@@ -346,7 +349,9 @@ class DARTsortGroundTruthComparison:
346
349
  matched_gt_mask = gt_spike_labels == "TP"
347
350
  matched_gt_indices = in_gt_unit[matched_gt_mask]
348
351
  only_gt_indices = in_gt_unit[np.logical_not(matched_gt_mask)]
349
- return in_gt_unit, matched_gt_indices, only_gt_indices
352
+ ret = in_gt_unit, matched_gt_indices, only_gt_indices
353
+ self._matched_and_missed_cache[gt_unit] = ret
354
+ return ret
350
355
 
351
356
  def get_spikes_by_category(self, gt_unit, tested_unit=None):
352
357
  if tested_unit is None:
@@ -88,8 +88,10 @@ class GrabAndFeaturize(BasePeeler):
88
88
  skip_features: bool = False,
89
89
  n_resid_snips: int | None = None,
90
90
  to_cpu: bool = True,
91
+ **peel_kw,
91
92
  ) -> "PeelingBatchResult":
92
93
  """Override process_chunk to skip empties."""
94
+ assert not peel_kw
93
95
  if chunk_end_samples is None:
94
96
  chunk_end_samples = min(
95
97
  self.recording.get_num_samples(),
@@ -24,7 +24,7 @@ from ..util.internal_config import (
24
24
  )
25
25
  from ..util.logging_util import get_logger
26
26
  from ..util.motion import MotionInfo
27
- from ..util.waveform_util import make_channel_index
27
+ from ..util.waveform_util import full_channel_index, make_channel_index
28
28
  from .matching_util import (
29
29
  ChunkTemplateData,
30
30
  MatchingPeaks,
@@ -260,6 +260,7 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
260
260
  return_residual=False,
261
261
  return_waveforms=True,
262
262
  return_conv=False,
263
+ return_clean_waveforms=False,
263
264
  ) -> PeelingBatchResult:
264
265
  assert self.matching_templates is not None
265
266
  # get chunk center time and template info at that time
@@ -288,6 +289,7 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
288
289
  return_residual=return_residual,
289
290
  return_conv=return_conv,
290
291
  return_collisioncleaned_waveforms=return_waveforms,
292
+ return_clean_waveforms=return_clean_waveforms,
291
293
  )
292
294
 
293
295
  # process spike times and create return result
@@ -304,9 +306,11 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
304
306
  self,
305
307
  traces: Tensor,
306
308
  chunk_template_data: ChunkTemplateData,
309
+ *,
307
310
  left_margin=0,
308
311
  right_margin=0,
309
312
  return_collisioncleaned_waveforms=True,
313
+ return_clean_waveforms=False,
310
314
  return_residual=False,
311
315
  return_conv=False,
312
316
  unit_mask=None,
@@ -475,6 +479,14 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
475
479
  assert self.p.channel_selection == "template"
476
480
  channels = chunk_template_data.main_channels[peaks.template_inds]
477
481
  waveforms = None
482
+ if return_clean_waveforms:
483
+ ci_full = full_channel_index(self.b.channel_index.shape[0], True)
484
+ ci_full = ci_full.to(self.b.channel_index)
485
+ res["clean_waveforms"] = chunk_template_data.get_clean_waveforms(
486
+ peaks=peaks,
487
+ channels=channels,
488
+ channel_index=ci_full,
489
+ )
478
490
  res["channels"] = channels
479
491
  if return_collisioncleaned_waveforms:
480
492
  assert waveforms is not None
@@ -9,6 +9,7 @@ from pathlib import Path
9
9
  from sys import getrefcount
10
10
  from threading import Lock, local
11
11
  from typing import Any, Sequence, TypedDict
12
+ from warnings import catch_warnings, filterwarnings
12
13
 
13
14
  import h5py
14
15
  import numpy as np
@@ -496,6 +497,7 @@ class BasePeeler(BModule):
496
497
  skip_features: bool = False,
497
498
  n_resid_snips: int | None = None,
498
499
  to_cpu: bool = True,
500
+ **peel_kw,
499
501
  ) -> "PeelingBatchResult":
500
502
  """Grab, peel, and featurize a chunk, returning a dict of numpy arrays
501
503
 
@@ -512,6 +514,7 @@ class BasePeeler(BModule):
512
514
  right_margin=right_margin,
513
515
  return_waveforms=return_waveforms,
514
516
  return_residual=return_residual or bool(n_resid_snips),
517
+ **peel_kw,
515
518
  )
516
519
  chunk_result = self.featurize_chunk_result(
517
520
  peel_result=peel_result,
@@ -537,8 +540,17 @@ class BasePeeler(BModule):
537
540
  channel_indices=None,
538
541
  margin=self.chunk_margin_samples,
539
542
  )
543
+ assert isinstance(chunk, np.ndarray)
540
544
  device = self.b.channel_index.device
541
- chunk = torch.tensor(chunk, device=device, dtype=self.dtype)
545
+ if device.type == "cpu":
546
+ chunk = torch.tensor(chunk, device=device, dtype=self.dtype)
547
+ else:
548
+ # we have to copy anyway to go to device, so working around the
549
+ # torch warning here. this way, can avoid blocking.
550
+ with catch_warnings():
551
+ filterwarnings("ignore", message="The given NumPy array is not ")
552
+ chunk = torch.from_numpy(chunk)
553
+ chunk = chunk.to(device=device, dtype=self.dtype, non_blocking=True)
542
554
  return chunk, chunk_end_samples, left_margin, right_margin
543
555
 
544
556
  def featurize_chunk_result(
@@ -20,6 +20,7 @@ from ..util.internal_config import (
20
20
  )
21
21
  from ..util.job_util import ensure_computation_config
22
22
  from ..util.spiketorch import grab_spikes, ptp, subtract_spikes_
23
+ from ..util.torch_util import torch_compile
23
24
  from ..util.waveform_util import get_relative_subset, make_channel_index
24
25
  from .peel_base import PeelingBatchResult
25
26
 
@@ -89,46 +90,32 @@ def check_residual_decrease(
89
90
  return None, {}
90
91
  assert orig_wfs is not None
91
92
 
92
- if local_whiteners is not None:
93
- assert channels is not None
94
- W = local_whiteners[channels]
95
-
96
- # remove nans
97
- if overwrite_orig_waveforms:
98
- orig_wfs = orig_wfs.nan_to_num_()
99
- else:
100
- orig_wfs = orig_wfs.nan_to_num()
101
- dn_wfs = dn_wfs.nan_to_num()
102
-
103
- # spatial mul -- putting temporal dim last here
104
- orig_wfs = W.bmm(orig_wfs.mT)
105
- dn_wfs = W.bmm(dn_wfs.mT)
106
-
107
- # temporal conv if needed
108
- if whitening_kernel is not None:
109
- *shp, t = orig_wfs.shape
110
- k = whitening_kernel[None, None]
111
- orig_wfs = F.conv1d(orig_wfs.view(-1, 1, t), k, padding="same")
112
- dn_wfs = F.conv1d(dn_wfs.view(-1, 1, t), k, padding="same")
113
- orig_wfs = orig_wfs.view(*shp, t)
114
- dn_wfs = dn_wfs.view(*shp, t)
93
+ orig_wfs = flatten_denan_and_whiten_batched(
94
+ orig_wfs,
95
+ channels,
96
+ local_whiteners,
97
+ whitening_kernel,
98
+ overwrite=overwrite_orig_waveforms,
99
+ )
100
+ dn_wfs = flatten_denan_and_whiten_batched(
101
+ dn_wfs, channels, local_whiteners, whitening_kernel
102
+ )
115
103
 
116
104
  if decrease_objective == "deconv":
117
105
  if overwrite_orig_waveforms:
118
- buf = orig_wfs.mul_(dn_wfs).nan_to_num_()
119
- conv = buf.sum(dim=(1, 2))
106
+ buf = orig_wfs.mul_(dn_wfs)
107
+ conv = buf.sum(dim=1)
120
108
  torch.square(dn_wfs, out=buf)
121
- norm = buf.nan_to_num_().sum(dim=(1, 2))
109
+ norm = buf.sum(dim=1)
122
110
  else:
123
- dn_wfs = dn_wfs.nan_to_num()
124
- conv = (orig_wfs * dn_wfs).nan_to_num_().sum(dim=(1, 2))
125
- norm = dn_wfs.square_().sum(dim=(1, 2))
111
+ conv = (orig_wfs * dn_wfs).sum(dim=1)
112
+ norm = dn_wfs.square_().sum(dim=1)
126
113
  reduction = conv.mul_(2.0).sub_(norm)
127
114
  threshold = threshold**2
128
115
  elif decrease_objective in ("norm", "normsq"):
129
- orig_decobj = orig_wfs.square().sum(dim=(1, 2))
130
- orig_wfs = orig_wfs.sub_(dn_wfs).nan_to_num_()
131
- new_decobj = orig_wfs.square_().sum(dim=(1, 2))
116
+ orig_decobj = orig_wfs.square().sum(dim=1)
117
+ orig_wfs = orig_wfs.sub_(dn_wfs)
118
+ new_decobj = orig_wfs.square_().sum(dim=1)
132
119
  if decrease_objective == "norm":
133
120
  orig_decobj = orig_decobj.sqrt_()
134
121
  new_decobj = new_decobj.sqrt_()
@@ -147,6 +134,51 @@ def check_residual_decrease(
147
134
  return keep, features
148
135
 
149
136
 
137
+ @torch_compile
138
+ def flatten_denan_and_whiten_batched(
139
+ wfs: torch.Tensor,
140
+ channels: torch.Tensor | None,
141
+ local_whiteners: Tensor | None,
142
+ whitening_kernel: Tensor | None,
143
+ overwrite: bool = False,
144
+ batch_size: int = 4096,
145
+ ):
146
+ N = wfs.shape[0]
147
+ T = wfs.shape[1]
148
+ C = wfs.shape[2]
149
+ if overwrite:
150
+ wfs = wfs.nan_to_num_()
151
+ else:
152
+ wfs = wfs.nan_to_num()
153
+
154
+ if local_whiteners is None:
155
+ return wfs.view(N, -1)
156
+
157
+ assert channels is not None
158
+
159
+ W = local_whiteners[channels]
160
+ wfs = W.bmm(wfs.mT)
161
+ if whitening_kernel is None:
162
+ return wfs.view(N, -1)
163
+
164
+ k = whitening_kernel[None, None]
165
+ NC = N * C
166
+ wfs = wfs.view(NC, 1, T)
167
+
168
+ for i0 in range(0, NC, batch_size):
169
+ i1 = min(NC, i0 + batch_size)
170
+ nb = i1 - i0
171
+ y = wfs[i0:i1]
172
+ if nb < batch_size:
173
+ y = F.pad(y, (0, 0, 0, 0, 0, batch_size - nb))
174
+ z = F.conv1d(y, k, padding="same")
175
+ if nb < batch_size:
176
+ z = z[:nb]
177
+ wfs[i0:i1] = z
178
+
179
+ return wfs.view(N, -1)
180
+
181
+
150
182
  ChunkSubtractionResult = namedtuple(
151
183
  "ChunkSubtractionResult",
152
184
  [