DARTsort 0.5.6__tar.gz → 0.5.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. {dartsort-0.5.6 → dartsort-0.5.7}/PKG-INFO +1 -1
  2. {dartsort-0.5.6 → dartsort-0.5.7}/pyproject.toml +1 -0
  3. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/agglomerate.py +246 -1
  4. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/cluster_util.py +41 -4
  5. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/kmeans.py +2 -0
  6. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/mixture.py +76 -24
  7. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/modes.py +3 -42
  8. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/refine_util.py +1 -1
  9. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/config.py +12 -3
  10. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/analysis.py +39 -15
  11. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/comparison.py +26 -8
  12. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/hybrid_util.py +1 -1
  13. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/simkit.py +3 -1
  14. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/main.py +22 -2
  15. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching.py +2 -2
  16. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/pairwise_util.py +6 -0
  17. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/peel_base.py +1 -0
  18. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/reduction_template.py +1 -6
  19. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/get_templates.py +2 -1
  20. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/templates.py +49 -2
  21. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/_multichan_denoiser_kit.py +15 -2
  22. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/decollider.py +17 -3
  23. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/enforce_decrease.py +2 -1
  24. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/mixture_classifier.py +1 -1
  25. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/reduction.py +57 -45
  26. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/data_util.py +28 -10
  27. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/internal_config.py +32 -6
  28. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/interpolation_util.py +3 -3
  29. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/logging_util.py +4 -0
  30. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/nn_util.py +2 -0
  31. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/spiketorch.py +4 -0
  32. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/gt.py +39 -8
  33. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/mixture.py +2 -2
  34. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/recanim.py +2 -1
  35. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/scatterplots.py +18 -14
  36. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/unit.py +38 -6
  37. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/unit_comparison.py +34 -9
  38. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/versus.py +57 -9
  39. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/vismain.py +79 -8
  40. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/waveforms.py +5 -3
  41. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/PKG-INFO +1 -1
  42. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/SOURCES.txt +1 -0
  43. {dartsort-0.5.6 → dartsort-0.5.7}/tests/conftest.py +3 -3
  44. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_alignment.py +48 -0
  45. dartsort-0.5.7/tests/test_cluster_util.py +83 -0
  46. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_clustering.py +5 -5
  47. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_data_util.py +2 -1
  48. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_density.py +27 -10
  49. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_detect.py +3 -3
  50. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_enforce_decrease.py +2 -1
  51. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_modeling.py +13 -5
  52. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_spiketorch.py +2 -1
  53. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_torch_optimization_util.py +2 -1
  54. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_util.py +1 -0
  55. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_waveform_util.py +2 -2
  56. {dartsort-0.5.6 → dartsort-0.5.7}/.github/workflows/ci.yml +0 -0
  57. {dartsort-0.5.6 → dartsort-0.5.7}/.github/workflows/ci_windows.yml +0 -0
  58. {dartsort-0.5.6 → dartsort-0.5.7}/.github/workflows/deploy.yml +0 -0
  59. {dartsort-0.5.6 → dartsort-0.5.7}/.github/workflows/docs.yml +0 -0
  60. {dartsort-0.5.6 → dartsort-0.5.7}/.gitignore +0 -0
  61. {dartsort-0.5.6 → dartsort-0.5.7}/README.md +0 -0
  62. {dartsort-0.5.6 → dartsort-0.5.7}/docs/index.md +0 -0
  63. {dartsort-0.5.6 → dartsort-0.5.7}/docs/main_api.md +0 -0
  64. {dartsort-0.5.6 → dartsort-0.5.7}/docs/toolkit_api.md +0 -0
  65. {dartsort-0.5.6 → dartsort-0.5.7}/environment.yml +0 -0
  66. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/.keep +0 -0
  67. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/basic_integration_test.py +0 -0
  68. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/ci diag.py +0 -0
  69. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/dartsort_dredge_demo.ipynb +0 -0
  70. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/dartsort_dredge_demo.py +0 -0
  71. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/data test.py +0 -0
  72. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/dead reckoning.py +0 -0
  73. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/denoise np1.py +0 -0
  74. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/denoise np2.py +0 -0
  75. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/exploratory.py +0 -0
  76. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/geom debug.py +0 -0
  77. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/hdbscan_clustering_5min.py +0 -0
  78. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/hybrid 10.py +0 -0
  79. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/hybrid cluster.py +0 -0
  80. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/hybrid_11.py +0 -0
  81. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/initial training.py +0 -0
  82. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/make hybrid.py +0 -0
  83. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/mgh.py +0 -0
  84. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/np1 feats.py +0 -0
  85. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/np2 processing version c.py +0 -0
  86. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/paperfigs.py +0 -0
  87. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/pdists.py +0 -0
  88. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/postcheck.py +0 -0
  89. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/ptp vae.py +0 -0
  90. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/re clust.py +0 -0
  91. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/reg fig loop.py +0 -0
  92. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/reg_sim_exp.py +0 -0
  93. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/relocating.py +0 -0
  94. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/relocating_all.py +0 -0
  95. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/restats +0 -0
  96. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/sortStinger1.py +0 -0
  97. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/sortStinger3.py +0 -0
  98. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/sortStinger4.py +0 -0
  99. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/sortStinger5.py +0 -0
  100. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/sortStinger6.py +0 -0
  101. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/spike farm.py +0 -0
  102. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/ss_ensemble_sim.py +0 -0
  103. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/ss_ensemble_sim_drift.py +0 -0
  104. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/ss_sim.py +0 -0
  105. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/stan diagnostics.py +0 -0
  106. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/storytelling c.py +0 -0
  107. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/storytelling.py +0 -0
  108. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
  109. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/debug_matching.py +0 -0
  110. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_agreement.py +0 -0
  111. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_amplitude_scaling.py +0 -0
  112. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_batch_optim.py +0 -0
  113. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_ibme.py +0 -0
  114. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_mm_initialization.py +0 -0
  115. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_normxcorr1d.py +0 -0
  116. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_reassignment.py +0 -0
  117. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_relocation.py +0 -0
  118. {dartsort-0.5.6 → dartsort-0.5.7}/notebook/thomas.py +0 -0
  119. {dartsort-0.5.6 → dartsort-0.5.7}/setup.cfg +0 -0
  120. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/__init__.py +0 -0
  121. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/cli.py +0 -0
  122. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/__init__.py +0 -0
  123. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/clustering.py +0 -0
  124. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/clustering_features.py +0 -0
  125. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/density.py +0 -0
  126. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/forward_backward.py +0 -0
  127. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/merge.py +0 -0
  128. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/detect/__init__.py +0 -0
  129. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/detect/detect.py +0 -0
  130. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/detect/detect_filters.py +0 -0
  131. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/__init__.py +0 -0
  132. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  133. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  134. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/simlib.py +0 -0
  135. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/localize/__init__.py +0 -0
  136. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/localize/localize_torch.py +0 -0
  137. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/localize/localize_util.py +0 -0
  138. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/__init__.py +0 -0
  139. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/grab.py +0 -0
  140. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/__init__.py +0 -0
  141. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  142. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/drifty.py +0 -0
  143. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  144. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  145. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/peel_lib.py +0 -0
  146. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/shaver.py +0 -0
  147. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/subtract.py +0 -0
  148. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/threshold.py +0 -0
  149. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  150. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/detect_np1.pt +0 -0
  151. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/detect_np2.pt +0 -0
  152. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  153. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  154. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  155. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  156. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  157. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  158. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/__init__.py +0 -0
  159. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/postprocess_util.py +0 -0
  160. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/realignment.py +0 -0
  161. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/template_util.py +0 -0
  162. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/templib.py +0 -0
  163. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/__init__.py +0 -0
  164. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/all_transformers.py +0 -0
  165. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/amortized_localization.py +0 -0
  166. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/amplitudes.py +0 -0
  167. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/denoising_scorer.py +0 -0
  168. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/fixed_prop.py +0 -0
  169. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/interp.py +0 -0
  170. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/localize.py +0 -0
  171. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/matching_denoiser.py +0 -0
  172. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/pipeline.py +0 -0
  173. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  174. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/supervised_denoiser.py +0 -0
  175. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/temporal_pca.py +0 -0
  176. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/transform_base.py +0 -0
  177. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/whiten.py +0 -0
  178. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/__init__.py +0 -0
  179. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/cli_util.py +0 -0
  180. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/drift_util.py +0 -0
  181. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/job_util.py +0 -0
  182. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/main_util.py +0 -0
  183. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/more_operators.py +0 -0
  184. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/motion.py +0 -0
  185. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/multiprocessing_util.py +0 -0
  186. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/noise_util.py +0 -0
  187. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/peel_util.py +0 -0
  188. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/preprocess_util.py +0 -0
  189. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/py_util.py +0 -0
  190. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/registration_util.py +0 -0
  191. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/residual_util.py +0 -0
  192. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/sparse_util.py +0 -0
  193. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/spikeio.py +0 -0
  194. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/testing_util/__init__.py +0 -0
  195. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  196. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  197. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/torch_optimization_util.py +0 -0
  198. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/torch_util.py +0 -0
  199. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/waveform_util.py +0 -0
  200. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/__init__.py +0 -0
  201. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/analysis_plots.py +0 -0
  202. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/colors.py +0 -0
  203. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/layout.py +0 -0
  204. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/metrics_util.py +0 -0
  205. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/sorting.py +0 -0
  206. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/dependency_links.txt +0 -0
  207. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/entry_points.txt +0 -0
  208. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/requires.txt +0 -0
  209. {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/top_level.txt +0 -0
  210. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_config.py +0 -0
  211. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_dartsort.py +0 -0
  212. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_drift_util.py +0 -0
  213. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_drifty_matching.py +0 -0
  214. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_grab_and_featurize.py +0 -0
  215. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_kmeans.py +0 -0
  216. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_linalg.py +0 -0
  217. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_localize_torch.py +0 -0
  218. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_matching.py +0 -0
  219. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_mixture_classifier.py +0 -0
  220. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_simkit.py +0 -0
  221. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_subtract.py +0 -0
  222. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_templates.py +0 -0
  223. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_threshold.py +0 -0
  224. {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_transform.py +0 -0
  225. {dartsort-0.5.6 → dartsort-0.5.7}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.6
3
+ Version: 0.5.7
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -92,3 +92,4 @@ replace-imports-with-any = ["hdbscan"]
92
92
  [tool.ty.rules]
93
93
  "unsupported-base" = "ignore"
94
94
  "invalid-key" = "ignore"
95
+ "possibly-unresolved-reference" = "error"
@@ -177,7 +177,13 @@ def agglomerate(
177
177
  threshold=0.5,
178
178
  link=template_merge_cfg.linkage,
179
179
  )
180
- agg_sorting = reorder_by_depth(agg_sorting, motion=motion)
180
+
181
+ agg_sorting = combine_gmm_scores(agg_sorting, new_ids=new_ids)
182
+
183
+ agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
184
+
185
+ agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
186
+ new_ids = reorder[new_ids]
181
187
 
182
188
  return Agglomeration(
183
189
  agglomerated_sorting=agg_sorting,
@@ -625,3 +631,242 @@ def firing_corr(sorting: DARTsortSorting, dt: float, method="binsqrt"):
625
631
  fr = np.sqrt(fr.values)
626
632
 
627
633
  return np.corrcoef(fr, rowvar=False)
634
+
635
+
636
+ def combine_gmm_scores(
637
+ sorting: DARTsortSorting, new_ids: np.ndarray, old_prefix="gmm", new_prefix="merged"
638
+ ) -> DARTsortSorting:
639
+ """If new_ids merges units, return a sorting with merged likelihoods."""
640
+ candidates = getattr(sorting, f"{old_prefix}_candidates", None)
641
+ responsibilities = getattr(sorting, f"{old_prefix}_responsibilities", None)
642
+ logliks = getattr(sorting, f"{old_prefix}_log_liks", None)
643
+
644
+ havec = candidates is not None
645
+ haver = responsibilities is not None
646
+ havel = logliks is not None
647
+ assert all([havec, haver, havel]) or not any([havec, havel, haver])
648
+ if not havec:
649
+ return sorting
650
+ assert candidates is not None
651
+ assert responsibilities is not None
652
+ assert logliks is not None
653
+
654
+ # check that new_ids is a merge
655
+ assert (new_ids >= 0).all()
656
+ unique_new_ids, new_id_counts = np.unique(new_ids, return_counts=True)
657
+ assert unique_new_ids.shape[0] == unique_new_ids.max() + 1 <= new_ids.shape[0]
658
+ if unique_new_ids.shape == new_ids.shape:
659
+ return sorting.ephemeral_replace(
660
+ **{
661
+ f"{new_prefix}_candidates": candidates,
662
+ f"{new_prefix}_responsibilities": responsibilities,
663
+ f"{new_prefix}_logliks": logliks,
664
+ }
665
+ )
666
+
667
+ # check invariants at the top
668
+ if responsibilities.shape[1] > 2:
669
+ assert np.all(np.diff(responsibilities[:, :-1], axis=1) <= 0)
670
+ assert np.greater_equal(np.isneginf(logliks[:, :-1]), candidates == -1).all()
671
+ if sorting.labels is not None:
672
+ assert np.all(
673
+ np.logical_or(
674
+ sorting.labels < 0, sorting.labels == new_ids[candidates[:, 0]]
675
+ )
676
+ )
677
+
678
+ # two steps: first merge, then sort
679
+ # merge candidates
680
+ new_ids_ = np.pad(new_ids, [(0, 1)], constant_values=-1)
681
+ orig_bye = candidates < 0
682
+ nbye = orig_bye.sum()
683
+ cand = np.where(orig_bye, new_ids.shape[0], candidates)
684
+ cand = new_ids_[cand]
685
+ assert (cand < 0).sum() >= nbye
686
+
687
+ # deduplicate
688
+ mergedr = responsibilities[:, : cand.shape[1]].copy()
689
+ mergedl = logliks[:, : cand.shape[1]].copy()
690
+ _combine_loop(cand, new_id_counts, mergedr, mergedl)
691
+
692
+ # now re-sort
693
+ order = np.argsort(-mergedl, axis=1, kind="stable")
694
+ cand = np.take_along_axis(cand, axis=1, indices=order)
695
+ mergedr = np.take_along_axis(mergedr, axis=1, indices=order)
696
+ mergedl = np.take_along_axis(mergedl, axis=1, indices=order)
697
+ mergedl = np.concatenate([mergedl, logliks[:, cand.shape[1] :]], axis=1)
698
+ mergedr = np.concatenate([mergedr, responsibilities[:, cand.shape[1] :]], axis=1)
699
+
700
+ # check invariants at the bottom
701
+ if mergedr.shape[1] > 2:
702
+ assert np.all(np.diff(mergedr[:, : cand.shape[1]], axis=1) <= 0)
703
+ assert np.greater_equal(np.isneginf(mergedl[:, : cand.shape[1]]), cand == -1).all()
704
+ assert (cand < 0).sum() >= nbye
705
+ if sorting.labels is not None:
706
+ changed = sorting.labels != cand[:, 0]
707
+ changed = changed[sorting.labels >= 0]
708
+ logger.dartsortdebug(
709
+ f"Mixture component aggregation changes {100 * changed.mean():0.2f}"
710
+ f"% of spike labels ({changed.sum().item()} spikes)."
711
+ )
712
+ return sorting.ephemeral_replace(
713
+ labels=np.where(mergedl[:, 0] >= mergedl[:, -1], cand[:, 0], -1),
714
+ **{
715
+ f"{new_prefix}_candidates": cand,
716
+ f"{new_prefix}_responsibilities": mergedr,
717
+ f"{new_prefix}_logliks": mergedl,
718
+ },
719
+ )
720
+
721
+
722
+ @numba.njit(parallel=True)
723
+ def _combine_loop(
724
+ cand: np.ndarray,
725
+ new_id_counts: np.ndarray,
726
+ mergedr: np.ndarray,
727
+ mergedl: np.ndarray,
728
+ ):
729
+ for s in numba.prange(cand.shape[0]): # ty: ignore
730
+ rcand = cand[s]
731
+ for j in range(cand.shape[1] - 1):
732
+ ncandj = rcand[j]
733
+ if ncandj < 0 or new_id_counts[ncandj] <= 1:
734
+ continue
735
+
736
+ eq_ncandj = rcand[j + 1 :] == ncandj
737
+ if eq_ncandj.sum() <= 1:
738
+ continue
739
+
740
+ rsum = mergedr[s, j]
741
+ lsum = mergedl[s, j]
742
+ for i, k in enumerate(range(j + 1, cand.shape[1])):
743
+ if not eq_ncandj[i]:
744
+ continue
745
+ cand[s, k] = -1
746
+
747
+ if mergedl[s, k] == -np.inf:
748
+ continue
749
+ rsum += mergedr[s, k]
750
+ lsum = np.logaddexp(lsum, mergedl[s, k])
751
+
752
+ mergedr[s, k] = 0.0
753
+ mergedl[s, k] = -np.inf
754
+
755
+ mergedr[s, j] = rsum
756
+ mergedl[s, j] = lsum
757
+
758
+
759
+ def deduplicate_spikes(
760
+ sorting: DARTsortSorting,
761
+ radius_ms: float = -1.0,
762
+ score_by=("merged_logliks", "gmm_logliks", "scores"),
763
+ ) -> DARTsortSorting:
764
+ """The lower-scoring of any spikes within radius_samples of each other is relabeled to -1.
765
+
766
+ scores are grabbed from the first entry of score_by which is a valid property of the sorting.
767
+ If it's mu
768
+ """
769
+ if radius_ms < 0 or sorting.labels is None:
770
+ return sorting
771
+
772
+ radius_samples = WaveformConfig.ms_to_samples(
773
+ ms=radius_ms,
774
+ sampling_frequency=sorting.sampling_frequency,
775
+ )
776
+ assert radius_samples >= 0
777
+
778
+ new_labels = sorting.labels.copy()
779
+ scores = None
780
+ for sck in score_by:
781
+ scores = getattr(sorting, sck, None)
782
+ if scores is not None:
783
+ logger.dartsortdebug(f"deduplicate by score {sck}")
784
+ break
785
+ if scores is None:
786
+ raise ValueError(f"sorting had none of {score_by}.")
787
+ if scores.ndim == 2:
788
+ scores = scores[:, 0]
789
+ assert scores.ndim == 1
790
+ assert scores.shape == new_labels.shape
791
+
792
+ # handle unsorted times
793
+ tsort = np.argsort(sorting.times_samples)
794
+ new_labels = new_labels[tsort]
795
+ times_samples = sorting.times_samples[tsort]
796
+ scores = scores[tsort]
797
+
798
+ unit_ids = np.unique(new_labels)
799
+ unit_ids = unit_ids[unit_ids >= 0]
800
+ ndrop = 0
801
+ for unit_id in unit_ids:
802
+ in_unit = np.flatnonzero(new_labels == unit_id)
803
+ t = times_samples[in_unit]
804
+ dt = np.diff(t)
805
+ if dt.min() > radius_samples:
806
+ continue
807
+ discard = _dedup_unit(t, dt, scores[in_unit], radius_samples)
808
+ ndrop += discard.sum()
809
+ new_labels[in_unit[discard]] = -1
810
+
811
+ logger.dartsortdebug(f"drop {ndrop}/{len(sorting)} isi violator spikes")
812
+ new_labels = new_labels[np.argsort(tsort)]
813
+
814
+ return sorting.ephemeral_replace(labels=new_labels)
815
+
816
+
817
+ def _dedup_unit(
818
+ t: np.ndarray, dt: np.ndarray, scores: np.ndarray, radius: int
819
+ ) -> np.ndarray:
820
+ """Deduplicate a single unit's spike train."""
821
+ discard = np.zeros(t.shape, dtype=bool)
822
+ _dedup_unit_loop(dt, scores, radius, discard)
823
+ return discard
824
+
825
+
826
+ @numba.njit
827
+ def _dedup_unit_loop(
828
+ dt: np.ndarray, scores: np.ndarray, radius: int, discard: np.ndarray
829
+ ):
830
+ n = scores.shape[0]
831
+ i0 = 0
832
+ while i0 < n - 1:
833
+ if dt[i0] > radius:
834
+ i0 += 1
835
+ continue
836
+
837
+ i1 = i0 + 1
838
+ while i1 < n - 1 and dt[i1] <= radius:
839
+ i1 += 1
840
+
841
+ # now, i0:i1 + 1 is a slice of violators
842
+ # discard until all valid
843
+ score_slice = scores[i0 : i1 + 1]
844
+ dt_slice = dt[i0:i1]
845
+ order = np.argsort(score_slice)
846
+ for oo in order:
847
+ # discard spike i0 + oo
848
+ ii = i0 + oo
849
+ discard[ii] = True
850
+
851
+ # figure out isi after bridging the gap, handle edges
852
+ bridge_isi = 0.0
853
+ if ii < n - 1:
854
+ bridge_isi += dt[ii]
855
+ else:
856
+ bridge_isi = np.inf
857
+ if ii > 0:
858
+ bridge_isi += dt[ii - 1]
859
+ else:
860
+ bridge_isi = np.inf
861
+
862
+ # update dt with bridge isi
863
+ if ii < n - 1:
864
+ dt[ii] = bridge_isi
865
+ if ii > 0:
866
+ dt[ii - 1] = bridge_isi
867
+
868
+ # check if done
869
+ if dt_slice.min() > radius:
870
+ break
871
+
872
+ i0 = i1
@@ -321,8 +321,29 @@ def combine_disjoint(inds_a, labels_a, inds_b, labels_b):
321
321
 
322
322
 
323
323
  def reorder_by_depth(
324
- sorting, motion=None, spatial_footprints=None, geom=None, centroids=None
325
- ):
324
+ sorting: DARTsortSorting,
325
+ motion: MotionInfo | None = None,
326
+ spatial_footprints: np.ndarray | None = None,
327
+ geom: np.ndarray | None = None,
328
+ centroids: np.ndarray | None = None,
329
+ ) -> tuple[DARTsortSorting, np.ndarray]:
330
+ """Reorder cluster labels so that centroid depth is increasing
331
+
332
+ Parameters
333
+ ----------
334
+ sorting : DARTsortSorting
335
+ motion : MotionInfo | None, optional
336
+ spatial_footprints : np.ndarray | None, optional
337
+ geom : np.ndarray | None, optional
338
+ centroids : np.ndarray | None, optional
339
+
340
+ Returns
341
+ -------
342
+ reordered_sorting: DARTsortSorting
343
+ reorder: np.ndarray
344
+ reorder[j] is the new label of original unit j.
345
+ """
346
+ assert sorting.labels is not None
326
347
  kept = np.flatnonzero(sorting.labels >= 0)
327
348
  kept_labels = sorting.labels[kept]
328
349
 
@@ -355,8 +376,9 @@ def reorder_by_depth(
355
376
  # this one is some food for thought, lol.
356
377
  reorder = np.argsort(np.argsort(centroids, kind="stable"), kind="stable")
357
378
  labels[kept] = reorder[kept_labels]
379
+ reordered_sorting = sorting.ephemeral_replace(labels=labels)
358
380
 
359
- return sorting.ephemeral_replace(labels=labels)
381
+ return reordered_sorting, reorder
360
382
 
361
383
 
362
384
  def closest_registered_channels(
@@ -502,7 +524,22 @@ def get_main_channel_pcs(
502
524
  return features
503
525
 
504
526
 
505
- def decrumb(labels, min_size=5, in_place=False, flatten=True):
527
+ def decrumb(labels: np.ndarray, min_size: int=5, in_place=False, flatten=True):
528
+ """Remove small units
529
+
530
+ Parameters
531
+ ----------
532
+ labels : np.ndarray
533
+ min_size : int
534
+ in_place : bool
535
+ flatten : bool
536
+ Flatten the output label space to be contiguous.
537
+
538
+ Returns
539
+ -------
540
+ labels
541
+ The (flattened) decrumbed labels.
542
+ """
506
543
  kept = np.flatnonzero(labels >= 0)
507
544
  labels_kept = labels[kept]
508
545
  labels = labels if in_place else labels.copy()
@@ -99,6 +99,7 @@ def kmeanspp(
99
99
  )
100
100
  j = n_components
101
101
  else:
102
+ j = 0
102
103
  for j in range(1, n_components):
103
104
  if weights is not None:
104
105
  torch.mul(dists, weights, out=p)
@@ -176,6 +177,7 @@ def truncated_kmeans(
176
177
  it = progrange(n_initializations, desc="kmeans++")
177
178
  else:
178
179
  it = range(n_initializations)
180
+ _d = None
179
181
  for _ in it:
180
182
  _c, _l, _d, _p = kmeanspp(
181
183
  X,
@@ -1236,7 +1236,7 @@ class BatchedSpikeData:
1236
1236
  if self.candidates is None:
1237
1237
  return
1238
1238
  assert self.rg is not None
1239
- same_adj = _fill_blank_labels(
1239
+ same_adj, n_new = _fill_blank_labels(
1240
1240
  labels=self.candidates[:, 0],
1241
1241
  un_adj=self.un_adj,
1242
1242
  explore_adj=self.explore_adj,
@@ -1246,6 +1246,10 @@ class BatchedSpikeData:
1246
1246
  allow_uncovered=allow_uncovered,
1247
1247
  )
1248
1248
  assert same_adj is not None
1249
+ if allow_uncovered:
1250
+ assert n_new == 0
1251
+ else:
1252
+ n_units = n_units + n_new
1249
1253
  if not same_adj:
1250
1254
  logger.dartsortverbose(
1251
1255
  "_fill_blank_labels used explore adjacency in bootstrap_candidates."
@@ -1334,7 +1338,7 @@ class BatchedSpikeData:
1334
1338
  self.update_adjacency(n_units=None)
1335
1339
  assert self.candidates is not None
1336
1340
  assert self.rg is not None
1337
- res = _fill_blank_labels(
1341
+ res, n_new = _fill_blank_labels(
1338
1342
  labels=self.candidates[:, 0],
1339
1343
  un_adj=self.un_adj,
1340
1344
  explore_adj=self.explore_adj,
@@ -1347,6 +1351,7 @@ class BatchedSpikeData:
1347
1351
  max_steps=max_steps if max_steps is not None else self.coverage_steps,
1348
1352
  )
1349
1353
  assert res is None
1354
+ return n_new
1350
1355
 
1351
1356
  def erase_candidates(self):
1352
1357
  if self.candidates is not None:
@@ -1672,6 +1677,9 @@ class TruncatedSpikeData(BatchedSpikeData):
1672
1677
  )
1673
1678
  assert self.candidates is not None
1674
1679
  self.candidates[:, 0] = labels
1680
+ if pnoid:
1681
+ _k = labels.amax() + 1 + int((labels < 0).any())
1682
+ assert labels.unique().shape == (_k,)
1675
1683
  return self
1676
1684
 
1677
1685
  def update(
@@ -1846,6 +1854,7 @@ class TruncatedSpikeData(BatchedSpikeData):
1846
1854
  assert n_units_new <= n_units_orig
1847
1855
 
1848
1856
  if distances is not None:
1857
+ assert distances.shape[0] == distances.shape[1]
1849
1858
  assert distances.shape[0] == n_units_new
1850
1859
  assert distances.shape[0] <= n_units_orig
1851
1860
  assert remapping.mapping.max() + 1 == distances.shape[0]
@@ -1870,12 +1879,24 @@ class TruncatedSpikeData(BatchedSpikeData):
1870
1879
  out=self.candidates[:, : self.n_candidates],
1871
1880
  )
1872
1881
  self.candidates[:, self.n_candidates :] = -1
1882
+ if pnoid and distances is not None:
1883
+ assert self.candidates.amax() < distances.shape[0], 1
1873
1884
  self.update_adjacency(n_units_new)
1874
- self.ensure_coverage(
1885
+ n_new = self.ensure_coverage(
1875
1886
  allow_new_units=not allow_uncovered,
1876
1887
  allow_uncovered=allow_uncovered,
1877
1888
  max_steps=0 if allow_uncovered else None,
1878
1889
  )
1890
+ if allow_uncovered:
1891
+ assert n_new == 0
1892
+ elif distances is not None:
1893
+ d = distances.new_full(
1894
+ (n_units_new + n_new, n_units_new + n_new), torch.inf
1895
+ )
1896
+ d[:n_units_new, :n_units_new] = distances
1897
+ d[n_units_new:, n_units_new:].fill_diagonal_(0.0)
1898
+ if pnoid and distances is not None:
1899
+ assert self.candidates.amax() < distances.shape[0], 2
1879
1900
  if distances is not None:
1880
1901
  _, lut = self.update(new_top_candidates=None, distances=distances)
1881
1902
  else:
@@ -2515,6 +2536,7 @@ class TruncatedMixtureModel(BaseMixtureModel):
2515
2536
  assert self.b.noise_log_prop.isfinite().all()
2516
2537
  else:
2517
2538
  assert stats.noise_N is None
2539
+ lp = None
2518
2540
 
2519
2541
  # rank 0 case: mean only
2520
2542
  if self.signal_rank == 0:
@@ -2537,6 +2559,7 @@ class TruncatedMixtureModel(BaseMixtureModel):
2537
2559
  if pnoid:
2538
2560
  assert soln.isfinite().all()
2539
2561
  if not skip_proportions:
2562
+ assert lp is not None
2540
2563
  assert lp.isfinite().all()
2541
2564
 
2542
2565
  def fixed_weight_em(
@@ -4264,21 +4287,49 @@ def get_full_neighborhood_data(
4264
4287
 
4265
4288
  rg = np.random.default_rng(rg)
4266
4289
 
4267
- # indices that train/val splits will live inside (full split is not restricted)
4268
- n_fit = refinement_cfg.sampling_cfg.more_waveforms_fit
4269
- if fit_indices is None and len(sorting) > n_fit:
4270
- fit_indices = rg.choice(len(sorting), size=n_fit, replace=False)
4271
- fit_indices.sort()
4272
- elif fit_indices is None:
4273
- fit_indices = np.arange(len(sorting))
4274
- assert fit_indices is not None
4275
- n_fit = fit_indices.shape[0]
4276
-
4277
- # data splits: -1 full, 0 train, 2 val
4290
+ # how many waveforms will be used for fitting?
4291
+ n_fit = min(len(sorting), refinement_cfg.sampling_cfg.more_waveforms_fit)
4292
+ val_prop = refinement_cfg.val_proportion
4293
+ train_prop = 1.0 - val_prop
4294
+ fit_prop_of_total = n_fit / len(sorting)
4295
+
4296
+ # data splits: -1 full, 0 train, 1 val
4278
4297
  split_mask = np.full(len(sorting), -1, dtype=np.int8)
4279
- split_mask[fit_indices] = 0
4280
- n_val = int(np.ceil(refinement_cfg.val_proportion * n_fit))
4281
- split_mask[fit_indices[np.sort(rg.choice(n_fit, size=n_val, replace=False))]] = 1
4298
+ # these are stratified by unit in order to ensure that all units are represented
4299
+ # in at least the train set rather than risking a miss with totally random sampling
4300
+ assert sorting.labels is not None
4301
+ unit_ids, counts = np.unique(sorting.labels, return_counts=True)
4302
+ _pos = unit_ids >= 0
4303
+ counts = counts[_pos]
4304
+ unit_ids = unit_ids[_pos]
4305
+ fit_counts = np.minimum(
4306
+ counts, np.ceil(counts * fit_prop_of_total).astype(np.int64)
4307
+ )
4308
+ assert fit_counts.dtype.kind == "i"
4309
+ train_counts = np.maximum(train_prop * fit_counts, 1).astype(np.int64)
4310
+ for uid, train_count, fit_count in zip(unit_ids, train_counts, fit_counts):
4311
+ in_unit = np.flatnonzero(sorting.labels == uid)
4312
+ if in_unit.size > fit_count:
4313
+ fit_ixs = rg.choice(in_unit, size=fit_count)
4314
+ fit_ixs.sort()
4315
+ elif in_unit.size == fit_count:
4316
+ fit_ixs = in_unit
4317
+ else:
4318
+ assert False
4319
+ if fit_ixs.size > train_count:
4320
+ train_ixs = rg.choice(fit_ixs, size=train_count)
4321
+ train_ixs.sort()
4322
+ elif fit_ixs.size == train_count:
4323
+ train_ixs = fit_ixs
4324
+ else:
4325
+ assert False
4326
+
4327
+ # backfill with val set
4328
+ split_mask[fit_ixs] = 1
4329
+ # cover with train
4330
+ split_mask[train_ixs] = 0
4331
+
4332
+ # sparsify index sets
4282
4333
  train_indices = torch.asarray(np.flatnonzero(split_mask == 0))
4283
4334
  val_indices = torch.asarray(np.flatnonzero(split_mask == 1))
4284
4335
 
@@ -5944,9 +5995,9 @@ def _fill_blank_labels(
5944
5995
  (blank,) = (labels < 0).nonzero(as_tuple=True)
5945
5996
  Nblank = blank.numel()
5946
5997
  if not Nblank and ensure_coverage_only:
5947
- return None
5998
+ return None, 0
5948
5999
  elif not Nblank:
5949
- return True
6000
+ return True, 0
5950
6001
 
5951
6002
  # need full coverage of neighborhoods here. would prefer only to branch out with
5952
6003
  # explore as needed, so make those probs tiny.
@@ -5984,8 +6035,9 @@ def _fill_blank_labels(
5984
6035
  neighb_component_ids = neighb_component_ids - neighb_component_ids.amin()
5985
6036
  n_spikes = 0
5986
6037
  if pnoid:
5987
- Kold = labels.max() + 1 + int((labels < 0).any())
5988
- assert labels.unique().shape == (Kold,)
6038
+ Kold = labels.max().item() + 1 + int(Nblank > 0)
6039
+ _lshp = labels.unique().shape
6040
+ assert _lshp == (Kold,), f"{Kold=} {_lshp=}"
5989
6041
  for j, nid in enumerate(uncovered_neighbs):
5990
6042
  (in_nid,) = (neighborhood_ids == nid).nonzero(as_tuple=True)
5991
6043
  n_spikes += in_nid.numel()
@@ -5998,7 +6050,7 @@ def _fill_blank_labels(
5998
6050
  logger.dartsortverbose(
5999
6051
  f"_fill_blank_labels made {n_new_units} new units for {n_spikes} spikes."
6000
6052
  )
6001
- return None
6053
+ return None, n_new_units
6002
6054
 
6003
6055
  if uncovered:
6004
6056
  # raise if still_uncovered, warn or log otherwise
@@ -6011,7 +6063,7 @@ def _fill_blank_labels(
6011
6063
  )
6012
6064
 
6013
6065
  if ensure_coverage_only:
6014
- return None
6066
+ return None, n_new_units
6015
6067
 
6016
6068
  for i0 in range(0, Nblank, batch_size):
6017
6069
  i1 = min(Nblank, i0 + batch_size)
@@ -6021,7 +6073,7 @@ def _fill_blank_labels(
6021
6073
  draws = torch.multinomial(p, 1, replacement=True, generator=gen)
6022
6074
  labels[ii] = draws.view(ii.shape[0])
6023
6075
 
6024
- return keep_same_adj
6076
+ return keep_same_adj, n_new_units
6025
6077
 
6026
6078
 
6027
6079
  def _bootstrap_top(
@@ -42,46 +42,6 @@ def fit_unimodal_right(x, f, weights=None, cut=0, hard=False):
42
42
  return out
43
43
 
44
44
 
45
- def fit_truncnorm_right(x, f, weights=None, cut=0, hard=False, n_iter=10):
46
- """Like above, but fits truncated normal to xs > cut with MoM."""
47
- from scipy.stats import norm
48
-
49
- if weights is None:
50
- weights = np.ones_like(f)
51
-
52
- # figure out where cut lands and what f(cut) should be
53
- cuti_left = np.searchsorted(x, cut)
54
- if cuti_left >= len(x) - 2:
55
- # everything is to the left... return uniform
56
- return np.full_like(f, 1 / len(f))
57
- cuti_right = np.searchsorted(x, cut, side="right") - 1
58
- if cuti_right <= 1:
59
- # everything is to the right... fit normal!
60
- mean = np.average(x, weights=weights)
61
- var = np.average((x - mean) ** 2, weights=weights)
62
- return norm.pdf(x, loc=mean, scale=np.sqrt(var))
63
- # else, in the middle somewhere...
64
- assert cuti_left in (cuti_right, cuti_right + 1)
65
-
66
- xcut = x[cuti_right:]
67
- mean = np.average(xcut, weights=weights[cuti_right:])
68
- var = np.average((xcut - mean) ** 2, weights=weights[cuti_right:])
69
- mu = mean
70
- sigma = std = np.sqrt(var)
71
- for i in range(n_iter):
72
- alpha = (cut - mu) / sigma
73
- phi_alpha = norm.cdf(alpha)
74
- Z = 1.0 - phi_alpha
75
- # we have: mean ~ mu + sigma*phi(alpha)/Z
76
- # and var ~ sigma^2(1+alpha phi(alpha) /Z - (phi(alpha)/Z)^2)
77
- # implies mu = mean - sigma*phi(alpha)/Z
78
- # sigma^2 = var/(1+alpha phi(alpha) /Z - (phi(alpha)/Z)^2)
79
- sigma = np.sqrt(var / (1 + alpha * phi_alpha / Z - (phi_alpha / Z) ** 2))
80
- mu = mean - sigma * phi_alpha / Z
81
-
82
- return norm.pdf(x, loc=mu, scale=sigma)
83
-
84
-
85
45
  def fit_bimodal_at(x, f, weights=None, cut=0):
86
46
  from isosplit import up_down_isotonic_regression # type: ignore
87
47
 
@@ -157,6 +117,8 @@ def smoothed_dipscore_at(
157
117
 
158
118
  if score_kind == "ks":
159
119
  empirical = np.cumsum(densities * spacings)
120
+ else:
121
+ empirical = None
160
122
 
161
123
  if null in ("isotonic", "gcm"):
162
124
  orders = (slice(None),)
@@ -174,8 +136,6 @@ def smoothed_dipscore_at(
174
136
  dens = fit_unimodal_right(sign * s, d, cut=sign * cut, hard=hard)
175
137
  elif null == "isotonic":
176
138
  dens = up_down_isotonic_regression(d)
177
- elif null == "truncnorm":
178
- dens = fit_truncnorm_right(sign * s, d, cut=sign * cut, hard=hard)
179
139
  else:
180
140
  assert False
181
141
 
@@ -185,6 +145,7 @@ def smoothed_dipscore_at(
185
145
 
186
146
  dens_err = (np.abs(dens - densities) * spacings).sum()
187
147
  if score_kind == "ks":
148
+ assert empirical is not None
188
149
  my_score = np.abs(empirical - np.cumsum(dens * spacings)).max()
189
150
  elif score_kind == "tv":
190
151
  my_score = 0.5 * dens_err
@@ -175,7 +175,7 @@ def pc_merge(
175
175
  )
176
176
  sf = means.square_().sum(dim=1).sqrt_() * counts.sqrt()
177
177
  sf = sf.numpy(force=True)
178
- sorting = reorder_by_depth(
178
+ sorting, new_ids = reorder_by_depth(
179
179
  sorting, motion=motion, spatial_footprints=sf, geom=motion.rgeom
180
180
  )
181
181
  return PCMergeResult(sorting=sorting)
@@ -123,6 +123,13 @@ class DARTsortUserConfig:
123
123
  alignment_ms: Annotated[float, Field(gt=0)] = 1.5
124
124
  """Largest time shift allowed when re-aligning events."""
125
125
 
126
+ deduplication_ms: Annotated[float, Field(gt=0)] = 0.5
127
+ """As a final postprocessing step, only the higher-scoring of any spikes within
128
+ this time radius of each other are kept.
129
+
130
+ If this is negative, it does nothing. If it's 0, exact duplicates are dropped.
131
+ """
132
+
126
133
  # -- thresholds
127
134
  peak_sign: Literal["neg", "both", "pos"] = "both"
128
135
  """Allow only troughs or events of both signs when detecting threshold
@@ -239,7 +246,9 @@ class DeveloperConfig(DARTsortUserConfig):
239
246
  n_residual_snips: int = 4 * 4096
240
247
 
241
248
  # initial detection
242
- nn_denoiser_max_waveforms_fit: int = 250_000
249
+ nn_denoiser_max_waveforms_fit: int = 512_000
250
+ nn_denoiser_noise_waveforms: int = 100 * 256
251
+ nn_denoiser_extra_kwargs: dict | None = None
243
252
  do_tpca_denoise: bool = True
244
253
  first_denoiser_thinning: float = 0.0
245
254
  first_denoiser_spatial_dedup_radius: float = 100.0
@@ -295,8 +304,8 @@ class DeveloperConfig(DARTsortUserConfig):
295
304
  initial_pc_scale: float = 2.0
296
305
  initial_pc_pre_scale: float = 0.5
297
306
  motion_aware_clustering: bool = True
298
- clustering_max_spikes: Annotated[int, Field(gt=0)] = 500_000
299
- pre_refinement_merge: bool = False
307
+ clustering_max_spikes: Annotated[int, Field(gt=0)] = 1024 * 1000
308
+ pre_refinement_merge: bool = True
300
309
  post_refinement_merge: bool = False
301
310
  pre_refinement_merge_metric: str = "normeuc"
302
311
  pre_refinement_merge_threshold: float = 0.1