DARTsort 0.5.3__tar.gz → 0.5.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. {dartsort-0.5.3 → dartsort-0.5.5}/PKG-INFO +1 -1
  2. {dartsort-0.5.3 → dartsort-0.5.5}/docs/main_api.md +3 -2
  3. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/__init__.py +2 -2
  4. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/kmeans.py +2 -1
  5. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/mixture.py +43 -43
  6. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/config.py +1 -2
  7. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/hybrid_util.py +7 -7
  8. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/simkit.py +11 -7
  9. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/main.py +22 -15
  10. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/pairwise.py +2 -2
  11. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/reduction_template.py +8 -3
  12. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/postprocess_util.py +2 -2
  13. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/amortized_localization.py +3 -3
  14. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/pipeline.py +3 -3
  15. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/reduction.py +1 -1
  16. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/single_channel_denoiser.py +2 -2
  17. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/cli_util.py +0 -1
  18. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/data_util.py +21 -15
  19. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/internal_config.py +4 -2
  20. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/logging_util.py +25 -18
  21. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/main_util.py +14 -9
  22. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/motion.py +18 -9
  23. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/multiprocessing_util.py +9 -0
  24. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/noise_util.py +4 -3
  25. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/peel_util.py +2 -2
  26. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/preprocess_util.py +4 -0
  27. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/py_util.py +17 -6
  28. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/spiketorch.py +16 -6
  29. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/waveform_util.py +1 -1
  30. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/colors.py +2 -2
  31. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/mixture.py +100 -37
  32. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/scatterplots.py +4 -1
  33. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/unit.py +1 -0
  34. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/PKG-INFO +1 -1
  35. {dartsort-0.5.3 → dartsort-0.5.5}/tests/conftest.py +6 -6
  36. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_alignment.py +1 -1
  37. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_matching.py +1 -1
  38. {dartsort-0.5.3 → dartsort-0.5.5}/.github/workflows/ci.yml +0 -0
  39. {dartsort-0.5.3 → dartsort-0.5.5}/.github/workflows/ci_windows.yml +0 -0
  40. {dartsort-0.5.3 → dartsort-0.5.5}/.github/workflows/deploy.yml +0 -0
  41. {dartsort-0.5.3 → dartsort-0.5.5}/.github/workflows/docs.yml +0 -0
  42. {dartsort-0.5.3 → dartsort-0.5.5}/.gitignore +0 -0
  43. {dartsort-0.5.3 → dartsort-0.5.5}/README.md +0 -0
  44. {dartsort-0.5.3 → dartsort-0.5.5}/docs/index.md +0 -0
  45. {dartsort-0.5.3 → dartsort-0.5.5}/docs/toolkit_api.md +0 -0
  46. {dartsort-0.5.3 → dartsort-0.5.5}/environment.yml +0 -0
  47. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/.keep +0 -0
  48. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/basic_integration_test.py +0 -0
  49. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/ci diag.py +0 -0
  50. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/dartsort_dredge_demo.ipynb +0 -0
  51. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/dartsort_dredge_demo.py +0 -0
  52. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/data test.py +0 -0
  53. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/dead reckoning.py +0 -0
  54. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/denoise np1.py +0 -0
  55. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/denoise np2.py +0 -0
  56. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/exploratory.py +0 -0
  57. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/geom debug.py +0 -0
  58. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/hdbscan_clustering_5min.py +0 -0
  59. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/hybrid 10.py +0 -0
  60. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/hybrid cluster.py +0 -0
  61. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/hybrid_11.py +0 -0
  62. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/initial training.py +0 -0
  63. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/make hybrid.py +0 -0
  64. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/mgh.py +0 -0
  65. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/np1 feats.py +0 -0
  66. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/np2 processing version c.py +0 -0
  67. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/paperfigs.py +0 -0
  68. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/pdists.py +0 -0
  69. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/postcheck.py +0 -0
  70. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/ptp vae.py +0 -0
  71. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/re clust.py +0 -0
  72. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/reg fig loop.py +0 -0
  73. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/reg_sim_exp.py +0 -0
  74. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/relocating.py +0 -0
  75. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/relocating_all.py +0 -0
  76. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/restats +0 -0
  77. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/sortStinger1.py +0 -0
  78. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/sortStinger3.py +0 -0
  79. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/sortStinger4.py +0 -0
  80. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/sortStinger5.py +0 -0
  81. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/sortStinger6.py +0 -0
  82. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/spike farm.py +0 -0
  83. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/ss_ensemble_sim.py +0 -0
  84. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/ss_ensemble_sim_drift.py +0 -0
  85. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/ss_sim.py +0 -0
  86. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/stan diagnostics.py +0 -0
  87. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/storytelling c.py +0 -0
  88. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/storytelling.py +0 -0
  89. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
  90. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/debug_matching.py +0 -0
  91. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_agreement.py +0 -0
  92. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_amplitude_scaling.py +0 -0
  93. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_batch_optim.py +0 -0
  94. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_ibme.py +0 -0
  95. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_mm_initialization.py +0 -0
  96. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_normxcorr1d.py +0 -0
  97. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_reassignment.py +0 -0
  98. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_relocation.py +0 -0
  99. {dartsort-0.5.3 → dartsort-0.5.5}/notebook/thomas.py +0 -0
  100. {dartsort-0.5.3 → dartsort-0.5.5}/pyproject.toml +0 -0
  101. {dartsort-0.5.3 → dartsort-0.5.5}/setup.cfg +0 -0
  102. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/cli.py +0 -0
  103. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/__init__.py +0 -0
  104. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/agglomerate.py +0 -0
  105. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/cluster_util.py +0 -0
  106. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/clustering.py +0 -0
  107. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/clustering_features.py +0 -0
  108. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/density.py +0 -0
  109. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/forward_backward.py +0 -0
  110. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/merge.py +0 -0
  111. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/modes.py +0 -0
  112. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/refine_util.py +0 -0
  113. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/detect/__init__.py +0 -0
  114. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/detect/detect.py +0 -0
  115. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/detect/detect_filters.py +0 -0
  116. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/__init__.py +0 -0
  117. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/analysis.py +0 -0
  118. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/comparison.py +0 -0
  119. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  120. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  121. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/simlib.py +0 -0
  122. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/localize/__init__.py +0 -0
  123. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/localize/localize_torch.py +0 -0
  124. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/localize/localize_util.py +0 -0
  125. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/__init__.py +0 -0
  126. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/grab.py +0 -0
  127. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching.py +0 -0
  128. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/__init__.py +0 -0
  129. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  130. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/drifty.py +0 -0
  131. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  132. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  133. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/peel_base.py +0 -0
  134. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/peel_lib.py +0 -0
  135. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/shaver.py +0 -0
  136. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/subtract.py +0 -0
  137. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/threshold.py +0 -0
  138. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  139. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/detect_np1.pt +0 -0
  140. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/detect_np2.pt +0 -0
  141. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  142. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  143. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  144. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  145. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  146. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  147. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/__init__.py +0 -0
  148. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/get_templates.py +0 -0
  149. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/realignment.py +0 -0
  150. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/template_util.py +0 -0
  151. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/templates.py +0 -0
  152. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/templib.py +0 -0
  153. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/__init__.py +0 -0
  154. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  155. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/all_transformers.py +0 -0
  156. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/amplitudes.py +0 -0
  157. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/decollider.py +0 -0
  158. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/denoising_scorer.py +0 -0
  159. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/enforce_decrease.py +0 -0
  160. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/fixed_prop.py +0 -0
  161. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/interp.py +0 -0
  162. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/localize.py +0 -0
  163. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/matching_denoiser.py +0 -0
  164. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/mixture_classifier.py +0 -0
  165. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/supervised_denoiser.py +0 -0
  166. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/temporal_pca.py +0 -0
  167. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/transform_base.py +0 -0
  168. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/whiten.py +0 -0
  169. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/__init__.py +0 -0
  170. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/drift_util.py +0 -0
  171. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/interpolation_util.py +0 -0
  172. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/job_util.py +0 -0
  173. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/more_operators.py +0 -0
  174. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/nn_util.py +0 -0
  175. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/registration_util.py +0 -0
  176. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/residual_util.py +0 -0
  177. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/sparse_util.py +0 -0
  178. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/spikeio.py +0 -0
  179. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/testing_util/__init__.py +0 -0
  180. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  181. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  182. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/torch_optimization_util.py +0 -0
  183. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/torch_util.py +0 -0
  184. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/__init__.py +0 -0
  185. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/analysis_plots.py +0 -0
  186. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/gt.py +0 -0
  187. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/layout.py +0 -0
  188. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/metrics_util.py +0 -0
  189. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/recanim.py +0 -0
  190. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/sorting.py +0 -0
  191. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/unit_comparison.py +0 -0
  192. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/versus.py +0 -0
  193. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/vismain.py +0 -0
  194. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/waveforms.py +0 -0
  195. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/SOURCES.txt +0 -0
  196. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/dependency_links.txt +0 -0
  197. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/entry_points.txt +0 -0
  198. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/requires.txt +0 -0
  199. {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/top_level.txt +0 -0
  200. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_clustering.py +0 -0
  201. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_config.py +0 -0
  202. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_dartsort.py +0 -0
  203. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_data_util.py +0 -0
  204. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_detect.py +0 -0
  205. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_drift_util.py +0 -0
  206. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_drifty_matching.py +0 -0
  207. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_enforce_decrease.py +0 -0
  208. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_grab_and_featurize.py +0 -0
  209. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_kmeans.py +0 -0
  210. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_linalg.py +0 -0
  211. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_localize_torch.py +0 -0
  212. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_mixture_classifier.py +0 -0
  213. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_modeling.py +0 -0
  214. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_simkit.py +0 -0
  215. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_spiketorch.py +0 -0
  216. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_subtract.py +0 -0
  217. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_templates.py +0 -0
  218. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_threshold.py +0 -0
  219. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_torch_optimization_util.py +0 -0
  220. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_transform.py +0 -0
  221. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_util.py +0 -0
  222. {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_waveform_util.py +0 -0
  223. {dartsort-0.5.3 → dartsort-0.5.5}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.3
3
+ Version: 0.5.5
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -4,7 +4,8 @@ toc_depth: 3
4
4
 
5
5
  # Main API: `dartsort()` and the `DARTsortUserConfig`
6
6
 
7
- This page shows the main functions and objects that you'd run into when spike sorting with *dartsort*.
7
+ This page shows the main functions and objects that you'd run into when spike sorting with *dartsort*.
8
+ If you're just getting started, the [usage section on the front page](/#usage) might be good to read first.
8
9
 
9
10
  For details on parameters you should think about before running the sorter, see the [important configuration details note](/#important-configuration-details).
10
11
 
@@ -16,7 +17,7 @@ For details on parameters you should think about before running the sorter, see
16
17
  show_signature: true
17
18
  separate_signature: true
18
19
 
19
- The return value from the `dartsort()` function is a DARTsortReturn object, which is a dictionary containing spike trains and motion information:
20
+ The return value from the `dartsort()` function is a `DARTsortResult` object, which is a dictionary containing spike trains and motion information:
20
21
 
21
22
  ::: dartsort.DARTsortReturn
22
23
  options:
@@ -24,7 +24,7 @@ from .localize.localize_util import (
24
24
  from .main import (
25
25
  ObjectiveUpdateTemplateMatchingPeeler,
26
26
  SubtractionPeeler,
27
- DARTsortReturn,
27
+ DARTsortResult,
28
28
  check_recording,
29
29
  cluster,
30
30
  dartsort,
@@ -78,7 +78,7 @@ from .util.logging_util import (
78
78
  from .util.motion import MotionInfo, get_motion_info, try_load_motion_info
79
79
  from .util.noise_util import EmbeddedNoise
80
80
  from .util.preprocess_util import preprocess
81
- from .util.py_util import databag, resolve_path
81
+ from .util.py_util import databag, ensure_path
82
82
  from .util.waveform_util import full_channel_index, make_channel_index
83
83
 
84
84
  __version__ = importlib.metadata.version("dartsort")
@@ -22,7 +22,6 @@ from ..util.sparse_util import (
22
22
  sparse_centroid_distsq,
23
23
  )
24
24
  from ..util.spiketorch import spawn_torch_rg
25
- from .density import guess_mode
26
25
 
27
26
  logger = get_logger(__name__)
28
27
 
@@ -58,6 +57,8 @@ def kmeanspp(
58
57
  closest = torch.cdist(X, X.mean(0, keepdim=True)).argmax()
59
58
  centroid_ixs[0] = closest.item()
60
59
  elif kmeanspp_initial == "mode":
60
+ from .density import guess_mode
61
+
61
62
  Xm = X
62
63
  if Xm.shape[1] > mode_dim:
63
64
  q = min(mode_dim + 10, *Xm.shape)
@@ -220,9 +220,11 @@ def tmm_demix(
220
220
  )
221
221
  else:
222
222
  assert False
223
+
223
224
  stepname = f"tmm{outer_it}{inner_it}{step_type}"
224
- is_final = outer_it == refinement_cfg.n_total_iters - 1
225
- is_final = is_final and inner_it == len(refinement_cfg.mixture_steps) - 1
225
+ last_outer = outer_it == refinement_cfg.n_total_iters - 1
226
+ last_inner = inner_it == len(refinement_cfg.mixture_steps) - 1
227
+ is_final = last_outer and last_inner
226
228
  if saving and not is_final:
227
229
  save_tmm_labels(tmm=tmm, stepname=stepname, **save_kw) # type: ignore
228
230
 
@@ -1317,10 +1319,13 @@ class BatchedSpikeData:
1317
1319
  self.candidates.fill_(-1)
1318
1320
 
1319
1321
  def bootstrap_candidates(
1320
- self, distances: Tensor, un_adj_lut: NeighborhoodLUT | None = None
1322
+ self,
1323
+ distances: Tensor,
1324
+ un_adj_lut: NeighborhoodLUT | None = None,
1325
+ allow_uncovered: bool = False,
1321
1326
  ) -> NeighborhoodLUT:
1322
1327
  self.update_adjacency(n_units=distances.shape[0], un_adj_lut=un_adj_lut)
1323
- self._fill_missing(distances.shape[0])
1328
+ self._fill_missing(distances.shape[0], allow_uncovered=allow_uncovered)
1324
1329
 
1325
1330
  # fill in candidates[:, 1:n_candidates] at random obeying un_adj
1326
1331
  # choosing not to use distances here, since they get used in search sets
@@ -1859,7 +1864,7 @@ class TruncatedSpikeData(BatchedSpikeData):
1859
1864
  # have to do a full bootstrap, bc it's hard to figure out what to do with
1860
1865
  # spikes whose candidates contain the units that were split. this way, the
1861
1866
  # lut invariants are maintained, and at least the top labels are the same.
1862
- return self.bootstrap_candidates(distances)
1867
+ return self.bootstrap_candidates(distances, allow_uncovered=True)
1863
1868
 
1864
1869
  def full_proposal_view(self, un_adj_lut: NeighborhoodLUT):
1865
1870
  return FullProposalDataView.from_truncated_spike_data(self, un_adj_lut)
@@ -4050,6 +4055,10 @@ def get_truncated_datasets(
4050
4055
  device=device,
4051
4056
  rgeom=prgeom[:-1],
4052
4057
  )
4058
+
4059
+ gc.collect()
4060
+ torch.cuda.empty_cache()
4061
+
4053
4062
  assert isinstance(noise, EmbeddedNoise)
4054
4063
  noise.to(device=device)
4055
4064
  assert noise.rank == feature_rank
@@ -4152,6 +4161,9 @@ def get_truncated_datasets(
4152
4161
  else:
4153
4162
  assert False
4154
4163
 
4164
+ gc.collect()
4165
+ torch.cuda.empty_cache()
4166
+
4155
4167
  return neighb_cov, erp, train_data, val_data, full_data, noise, train_ixs, val_ixs
4156
4168
 
4157
4169
 
@@ -5197,7 +5209,7 @@ def try_kmeans(
5197
5209
  feature_rank: int,
5198
5210
  n_iter: int = 100,
5199
5211
  with_proportions: bool = True,
5200
- drop_prop: float = 0.025,
5212
+ drop_prop: float = 0.0,
5201
5213
  kmeanspp_initial="random",
5202
5214
  n_kmeans_tries: int = 25,
5203
5215
  n_kmeanspp_tries: int = 25,
@@ -5251,10 +5263,7 @@ def evaluate_group_demolitions(
5251
5263
  train_scores: Scores,
5252
5264
  eval_scores: Scores,
5253
5265
  ) -> GroupDemolition:
5254
- # grab relevant sections of train and eval scores
5255
5266
  group_ = group.to(train_scores.candidates)
5256
- in_group_train = torch.isin(train_scores.candidates, group_).any(dim=1)
5257
- (in_group_train,) = in_group_train.nonzero(as_tuple=True)
5258
5267
 
5259
5268
  # select which units in the group are candidates for demolition
5260
5269
  if mean_train_resp is None:
@@ -6059,36 +6068,30 @@ def _count_candidates(candidates, batch_candidate_counts, batch_size):
6059
6068
 
6060
6069
  if TORCH_IS_OLD:
6061
6070
 
6062
- @torch.jit.script
6063
- def _combine_similar_resps(resps: Tensor, keep_mask: Tensor, n_keep: int) -> Tensor:
6064
- n_discard = resps.shape[1] - n_keep
6065
- assert n_discard + n_keep == resps.shape[1]
6066
- discard_mask = torch.logical_not(keep_mask)
6067
- discard_ix = discard_mask.nonzero()[:, 0]
6068
- keep_ix = keep_mask.nonzero()[:, 0]
6069
- assert keep_ix.numel() + discard_ix.numel() == resps.shape[1]
6070
- kept_resp = resps[:, keep_ix]
6071
- discard_resp = resps[:, discard_ix]
6072
- sim = discard_resp.T @ kept_resp
6073
- match = sim.argmax(1)
6074
- kept_resp[:, match] += discard_resp
6075
- return kept_resp
6071
+ def _nonzero_static(x: Tensor, size: int):
6072
+ nz = x.nonzero()
6073
+ assert nz.numel() == size
6074
+ return nz
6076
6075
  else:
6077
6076
 
6078
- @torch.jit.script
6079
- def _combine_similar_resps(resps: Tensor, keep_mask: Tensor, n_keep: int) -> Tensor:
6080
- n_discard = resps.shape[1] - n_keep
6081
- assert n_discard + n_keep == resps.shape[1]
6082
- discard_mask = torch.logical_not(keep_mask)
6083
- discard_ix = discard_mask.nonzero_static(size=n_discard)[:, 0]
6084
- keep_ix = keep_mask.nonzero_static(size=n_keep)[:, 0]
6085
- assert keep_ix.numel() + discard_ix.numel() == resps.shape[1]
6086
- kept_resp = resps[:, keep_ix]
6087
- discard_resp = resps[:, discard_ix]
6088
- sim = discard_resp.T @ kept_resp
6089
- match = sim.argmax(1)
6090
- kept_resp[:, match] += discard_resp
6091
- return kept_resp
6077
+ def _nonzero_static(x: Tensor, size: int):
6078
+ return x.nonzero_static(size=size)
6079
+
6080
+
6081
+ @torch.jit.script
6082
+ def _combine_similar_resps(resps: Tensor, keep_mask: Tensor, n_keep: int) -> Tensor:
6083
+ n_discard = resps.shape[1] - n_keep
6084
+ assert n_discard + n_keep == resps.shape[1]
6085
+ discard_mask = torch.logical_not(keep_mask)
6086
+ discard_ix = _nonzero_static(discard_mask, size=n_discard)[:, 0]
6087
+ keep_ix = _nonzero_static(keep_mask, size=n_keep)[:, 0]
6088
+ assert keep_ix.numel() + discard_ix.numel() == resps.shape[1]
6089
+ kept_resp = resps[:, keep_ix]
6090
+ discard_resp = resps[:, discard_ix]
6091
+ sim = discard_resp.T @ kept_resp
6092
+ match = sim.argmax(1)
6093
+ kept_resp[:, match] += discard_resp
6094
+ return kept_resp
6092
6095
 
6093
6096
 
6094
6097
  def concatenate_scores(scoress: list[Scores]) -> Scores:
@@ -6201,10 +6204,7 @@ def mean_responsibilities(
6201
6204
  rsum_batch.zero_()
6202
6205
 
6203
6206
  nc = int(ncand[bix].item())
6204
- if TORCH_IS_OLD:
6205
- cii, cjj = (cand[i0:i1] >= 0).nonzero().T
6206
- else:
6207
- cii, cjj = (cand[i0:i1] >= 0).nonzero_static(size=nc).T
6207
+ cii, cjj = _nonzero_static(cand[i0:i1] >= 0, size=nc).T
6208
6208
 
6209
6209
  c = cand[i0:i1][cii, cjj]
6210
6210
  r = resp[i0:i1][cii, cjj].double()
@@ -6491,8 +6491,8 @@ def _sparsify_candidates(
6491
6491
  assert cpos.any(dim=1).all()
6492
6492
  if pnoid and static_size is not None:
6493
6493
  assert cpos.sum() == static_size
6494
- if static_size is not None and not TORCH_IS_OLD:
6495
- spike_ixs, candidate_ixs = cpos.nonzero_static(size=static_size).T
6494
+ if static_size is not None:
6495
+ spike_ixs, candidate_ixs = _nonzero_static(cpos, size=static_size).T
6496
6496
  else:
6497
6497
  spike_ixs, candidate_ixs = cpos.nonzero(as_tuple=True)
6498
6498
  neighb_ixs = neighborhood_ids[spike_ixs]
@@ -1,8 +1,7 @@
1
1
  from collections.abc import Sequence
2
- from typing import Annotated, Literal, get_args
2
+ from typing import Annotated, Literal
3
3
 
4
4
  from pydantic import Field
5
- from typing_extensions import Doc
6
5
 
7
6
  from .util.internal_config import (
8
7
  InterpKernel,
@@ -23,7 +23,7 @@ from ..util.data_util import DARTsortSorting
23
23
  from ..util.internal_config import ComputationConfig, unshifted_raw_template_cfg
24
24
  from ..util.logging_util import progbar
25
25
  from ..util.motion import MotionInfo
26
- from ..util.py_util import resolve_path
26
+ from ..util.py_util import ensure_path
27
27
  from . import analysis, comparison, simkit
28
28
 
29
29
  logger = getLogger(__name__)
@@ -406,7 +406,7 @@ def load_dartsort_step_sortings(
406
406
  use, although its not a guarantee... h5 locking... need to figure it out.
407
407
  """
408
408
  mtime_dt = mtime_gap_minutes * 60 if mtime_gap_minutes else 0
409
- sorting_dir = resolve_path(sorting_dir, strict=True)
409
+ sorting_dir = ensure_path(sorting_dir, strict=True)
410
410
  if detection_h5_path is None:
411
411
  for dh5n in detection_h5_names:
412
412
  detection_h5_path = cast(Path, sorting_dir / dh5n)
@@ -416,12 +416,12 @@ def load_dartsort_step_sortings(
416
416
  age = time.time() - detection_h5_path.stat().st_mtime
417
417
  if age < mtime_dt:
418
418
  continue
419
- h5s = [resolve_path(detection_h5_path)]
419
+ h5s = [ensure_path(detection_h5_path)]
420
420
  break
421
421
  else:
422
422
  h5s = []
423
423
  else:
424
- h5s = [resolve_path(detection_h5_path)]
424
+ h5s = [ensure_path(detection_h5_path)]
425
425
 
426
426
  for j in range(1, 100):
427
427
  mh5 = sorting_dir / f"matching{j}.h5"
@@ -430,7 +430,7 @@ def load_dartsort_step_sortings(
430
430
  if mtime_dt:
431
431
  if time.time() - mh5.stat().st_mtime < mtime_dt:
432
432
  break
433
- h5s.append(resolve_path(mh5))
433
+ h5s.append(ensure_path(mh5))
434
434
 
435
435
  # let's check that there is at least something to do...
436
436
  labels_npys = sorting_dir.glob("*_labels.npy")
@@ -464,9 +464,9 @@ def load_dartsort_step_sortings(
464
464
  )
465
465
  if no_npys and st0.labels is not None:
466
466
  if hasattr(st0, "template_inds"):
467
- yield(
467
+ yield (
468
468
  name_formatter(f"{h5.stem}_template"),
469
- st0.ephemeral_replace(labels=st0.template_inds)
469
+ st0.ephemeral_replace(labels=st0.template_inds),
470
470
  )
471
471
  yield name_formatter(h5.stem), st0
472
472
  continue
@@ -20,8 +20,8 @@ from ..templates import TemplateData
20
20
  from ..util.data_util import (
21
21
  DARTsortSorting,
22
22
  divide_randomly,
23
+ ensure_path,
23
24
  extract_random_snips,
24
- resolve_path,
25
25
  )
26
26
  from ..util.job_util import ensure_computation_config
27
27
  from ..util.logging_util import get_logger, progbar
@@ -108,7 +108,7 @@ def generate_simulation(
108
108
  pass
109
109
 
110
110
  if noise_recording_folder is not None:
111
- noise_recording_folder = resolve_path(noise_recording_folder)
111
+ noise_recording_folder = ensure_path(noise_recording_folder)
112
112
  else:
113
113
  assert noise_in_memory
114
114
  duration_samples = int(duration_seconds * sampling_frequency)
@@ -146,7 +146,7 @@ def generate_simulation(
146
146
  return
147
147
 
148
148
  if folder is not None:
149
- folder = resolve_path(folder)
149
+ folder = ensure_path(folder)
150
150
  else:
151
151
  assert no_save
152
152
 
@@ -214,7 +214,7 @@ def generate_simulation(
214
214
 
215
215
 
216
216
  def load_simulation(folder):
217
- folder = resolve_path(folder, strict=True)
217
+ folder = ensure_path(folder, strict=True)
218
218
  recording_dir = folder / "recording"
219
219
  templates_npz = folder / "templates.npz"
220
220
  sorting_h5 = folder / "dartsort_sorting.h5"
@@ -252,7 +252,9 @@ class InjectSpikesPreprocessor(BasePreprocessor):
252
252
  **simulation_kwargs,
253
253
  )
254
254
  )
255
- self.segment = cast(InjectSpikesPreprocessorSegment, self._recording_segments[0])
255
+ self.segment = cast(
256
+ InjectSpikesPreprocessorSegment, self._recording_segments[0]
257
+ )
256
258
 
257
259
  def basic_sorting(self) -> DARTsortSorting:
258
260
  return self.segment.basic_sorting()
@@ -406,7 +408,9 @@ class InjectSpikesPreprocessor(BasePreprocessor):
406
408
 
407
409
  # residual snippets
408
410
  if n_residual_snips:
409
- nrs_dset = h5.create_dataset("n_residuals", data=np.zeros((), dtype=np.int64))
411
+ nrs_dset = h5.create_dataset(
412
+ "n_residuals", data=np.zeros((), dtype=np.int64)
413
+ )
410
414
  residual = h5.create_dataset(
411
415
  "residual",
412
416
  shape=(n_residual_snips, *self.segment.wf_shape),
@@ -486,7 +490,7 @@ class InjectSpikesPreprocessor(BasePreprocessor):
486
490
  save_collidedness=False,
487
491
  chunk_len_s=0.5,
488
492
  ):
489
- folder = resolve_path(folder)
493
+ folder = ensure_path(folder)
490
494
  folder.mkdir(exist_ok=True)
491
495
  recording_dir = folder / "recording"
492
496
  templates_npz = folder / "templates.npz"
@@ -62,13 +62,13 @@ from .util.motion import MotionInfo, get_motion_info
62
62
  from .util.noise_util import SpatialWhitener
63
63
  from .util.peel_util import run_peeler
64
64
  from .util.preprocess_util import preprocess
65
- from .util.py_util import dartcopytree, resolve_path, timer
65
+ from .util.py_util import dartcopytree, ensure_path, timer
66
66
  from .util.torch_util import cleanup_and_log_gpu_usage
67
67
 
68
68
  logger = get_logger(__name__)
69
69
 
70
70
 
71
- class DARTsortReturn(TypedDict):
71
+ class DARTsortResult(TypedDict):
72
72
  sorting: DARTsortSorting
73
73
  """Output spike trains."""
74
74
  motion: MotionInfo
@@ -86,21 +86,25 @@ def dartsort(
86
86
  dredge_motion_est: MotionEstimate | None = None,
87
87
  overwrite=False,
88
88
  ):
89
- """This function runs a spike sorter called dartsort.
89
+ """This function runs a spike sorter called *dartsort*.
90
90
 
91
91
  Parameters
92
92
  ---------
93
93
  recording : BaseRecording
94
- A SpikeInterface `BaseRecording` object
94
+ A SpikeInterface `BaseRecording`
95
95
  output_dir : str or Path
96
- Folder where outputs are stored
96
+ Folder where outputs are stored. See the `work_in_tmpdir` and `tmpdir_parent`
97
+ configuration options to store intermediate data in a scratch folder and
98
+ then only save the final outputs here.
97
99
  cfg : DARTsortUserConfig or DARTsortInternalConfig or str or Path
98
100
  Your settings. Either create a `DARTsortUserConfig` directly in code, or
99
101
  you can pass a string or Path pointing to a .toml file here.
100
102
  si_motion : spikeinterface.core.Motion, optional
101
- Allows users to pass their own external motion estimate.
103
+ Allows users to pass their own external motion estimate. If this is given,
104
+ the do_motion_estimation configuration flag is ignored and this object is
105
+ used.
102
106
  dredge_motion_est : dredge.MotionEstimate, optional
103
- Allows users to pass their own external motion estimate.
107
+ As in `si_motion`.
104
108
  overwrite : bool
105
109
  Ignore and overwrite stored results, if any. Otherwise, dartsort will
106
110
  try to resume from the last step that ran, or if it had finished then
@@ -108,13 +112,13 @@ def dartsort(
108
112
 
109
113
  Returns
110
114
  -------
111
- results : DARTsortReturn
115
+ results : DARTsortResult
112
116
  Dictionary of sorting results, with keys:
113
117
 
114
118
  - "sorting": `DARTsortSorting`
115
119
  - "motion": MotionInfo
116
120
  """
117
- output_dir = resolve_path(output_dir)
121
+ output_dir = ensure_path(output_dir)
118
122
  output_dir.mkdir(exist_ok=True)
119
123
 
120
124
  # convert cfg to internal format and store it for posterity
@@ -131,7 +135,7 @@ def dartsort(
131
135
  if cfg.work_in_tmpdir:
132
136
  with TemporaryDirectory(prefix="dartsort", dir=cfg.tmpdir_parent) as work_dir:
133
137
  # copy files and possibly recording to temporary directory
134
- work_dir = resolve_path(work_dir)
138
+ work_dir = ensure_path(work_dir)
135
139
  logger.dartsortdebug(f"Working in {work_dir}, outputs to {output_dir}.")
136
140
  recording, work_dir = ds_all_to_workdir(
137
141
  internal_cfg=cfg,
@@ -281,6 +285,7 @@ def _dartsort_impl(
281
285
  sampling_cfg=cfg.peeler_sampling_cfg,
282
286
  waveform_cfg=cfg.waveform_cfg,
283
287
  overwrite=overwrite,
288
+ _saving_intermediates=cfg.save_intermediate_features,
284
289
  )
285
290
  ret["motion"] = motion
286
291
  ds_save_motion(motion, output_dir, work_dir, overwrite)
@@ -301,6 +306,7 @@ def _dartsort_impl(
301
306
  sorting,
302
307
  motion=motion,
303
308
  refinement_cfgs=r_cfgs,
309
+ computation_cfg=cfg.computation_cfg,
304
310
  clustering_cfg=cfg.clustering_cfg,
305
311
  clustering_features_cfg=cfg.clustering_features_cfg,
306
312
  _save_cfg=cfg,
@@ -372,6 +378,7 @@ def _dartsort_impl(
372
378
  recording=recording,
373
379
  sorting=sorting,
374
380
  motion=motion,
381
+ computation_cfg=cfg.computation_cfg,
375
382
  refinement_cfgs=step_ref_cfgs,
376
383
  clustering_cfg=step_clus_cfg,
377
384
  clustering_features_cfg=step_clfeat_cfg,
@@ -390,7 +397,7 @@ def _dartsort_impl(
390
397
 
391
398
  # finally handle scratch directory and delete intermediate files if requested
392
399
  if work_dir is not None:
393
- orig_h5_path = resolve_path(sorting.parent_h5_path, strict=True)
400
+ orig_h5_path = ensure_path(sorting.parent_h5_path, strict=True)
394
401
  final_h5_path = output_dir / orig_h5_path.name
395
402
  assert final_h5_path.exists()
396
403
  sorting.parent_h5_path = final_h5_path
@@ -483,7 +490,7 @@ def subtract(
483
490
  hdf5_filename="subtraction.h5",
484
491
  model_subdir="subtraction_models",
485
492
  ) -> DARTsortSorting | None:
486
- output_dir = resolve_path(output_dir)
493
+ output_dir = ensure_path(output_dir)
487
494
  computation_cfg = ensure_computation_config(computation_cfg)
488
495
  check_recording(recording)
489
496
  subtraction_peeler = SubtractionPeeler.from_config(
@@ -544,7 +551,7 @@ def match(
544
551
  template_denoising_tsvd=None,
545
552
  whitener: SpatialWhitener | None = None,
546
553
  ) -> DARTsortSorting:
547
- output_dir = resolve_path(output_dir)
554
+ output_dir = ensure_path(output_dir)
548
555
  model_dir = output_dir / model_subdir
549
556
  computation_cfg = ensure_computation_config(computation_cfg)
550
557
 
@@ -632,7 +639,7 @@ def grab(
632
639
  model_subdir="grab_models",
633
640
  computation_cfg: ComputationConfig | None = None,
634
641
  ) -> DARTsortSorting:
635
- output_dir = resolve_path(output_dir)
642
+ output_dir = ensure_path(output_dir)
636
643
  grabber = GrabAndFeaturize.from_config(
637
644
  sorting=sorting,
638
645
  recording=recording,
@@ -672,7 +679,7 @@ def threshold(
672
679
  model_subdir="threshold_models",
673
680
  computation_cfg: ComputationConfig | None = None,
674
681
  ) -> DARTsortSorting:
675
- output_dir = resolve_path(output_dir)
682
+ output_dir = ensure_path(output_dir)
676
683
  computation_cfg = ensure_computation_config(computation_cfg)
677
684
  thresholder = Threshold.from_config(
678
685
  recording=recording,
@@ -10,7 +10,7 @@ from ...templates.template_util import CompressedUpsampledTemplates, LowRankTemp
10
10
  from ...templates.templates import TemplateData
11
11
  from ...util import job_util
12
12
  from ...util.motion import MotionInfo
13
- from ...util.py_util import resolve_path
13
+ from ...util.py_util import ensure_path
14
14
  from .matching_base import PconvBase
15
15
  from .pairwise_util import compressed_convolve_to_h5
16
16
 
@@ -149,7 +149,7 @@ class CompressedPairwiseConv(PconvBase):
149
149
  if computation_cfg is None:
150
150
  computation_cfg = job_util.get_global_computation_config()
151
151
 
152
- hdf5_filename = resolve_path(hdf5_filename)
152
+ hdf5_filename = ensure_path(hdf5_filename)
153
153
  hdf5_filename.parent.mkdir(exist_ok=True)
154
154
 
155
155
  # TODO: rewrite.
@@ -1,4 +1,5 @@
1
1
  """A peeler implementing mean or median reduction for estimating template waveforms."""
2
+
2
3
  from dataclasses import replace
3
4
  from pathlib import Path
4
5
  from tempfile import TemporaryDirectory
@@ -31,7 +32,7 @@ from ..util.job_util import ensure_computation_config
31
32
  from ..util.logging_util import get_logger
32
33
  from ..util.motion import MotionInfo
33
34
  from ..util.noise_util import SpatialWhitener
34
- from ..util.py_util import resolve_path
35
+ from ..util.py_util import ensure_path
35
36
  from ..util.waveform_util import full_channel_index
36
37
  from .grab import GrabAndFeaturize
37
38
 
@@ -58,6 +59,7 @@ class ReductionTemplateData(TemplateData):
58
59
  computation_cfg: ComputationConfig | None = None,
59
60
  show_progress: bool = True,
60
61
  ) -> TemplateData:
62
+ computation_cfg = ensure_computation_config(computation_cfg)
61
63
  # subsample sorting
62
64
  sorting = subsample_by_count_and_valid_time(
63
65
  sorting,
@@ -75,24 +77,25 @@ class ReductionTemplateData(TemplateData):
75
77
  motion=motion,
76
78
  waveform_cfg=waveform_cfg,
77
79
  template_cfg=template_cfg,
80
+ computation_cfg=computation_cfg,
78
81
  whitener=whitener,
79
82
  tsvd=tsvd,
80
83
  )
81
84
 
82
85
  # TODO: file not always needed
83
- computation_cfg = ensure_computation_config(computation_cfg)
84
86
  if template_cfg.reduction == "mean":
85
87
  # TODO: reducer doesn't work in parallel when gathering means, go to single job
86
88
  computation_cfg = ComputationConfig(
87
89
  device=computation_cfg.actual_device().type,
88
90
  n_jobs_small=computation_cfg.n_jobs_small,
91
+ tmpdir_parent=computation_cfg.tmpdir_parent,
89
92
  )
90
93
  with TemporaryDirectory(
91
94
  prefix="dartsorttemplates",
92
95
  ignore_cleanup_errors=True,
93
96
  dir=computation_cfg.tmpdir_parent,
94
97
  ) as tdir:
95
- tdir = resolve_path(tdir)
98
+ tdir = ensure_path(tdir)
96
99
  h5p = tdir / "tmp.h5"
97
100
  p.load_or_fit_and_save_models(tdir / "models")
98
101
  if template_cfg.denoising_method == "none" and not template_cfg.use_svd:
@@ -190,6 +193,7 @@ class TemplateReduction(GrabAndFeaturize):
190
193
  sorting: DARTsortSorting,
191
194
  waveform_cfg: WaveformConfig,
192
195
  template_cfg: TemplateConfig,
196
+ computation_cfg: ComputationConfig,
193
197
  whitener: SpatialWhitener | None = None,
194
198
  ):
195
199
  # geom processing
@@ -228,6 +232,7 @@ class TemplateReduction(GrabAndFeaturize):
228
232
  motion=motion,
229
233
  template_cfg=template_cfg,
230
234
  waveform_cfg=waveform_cfg,
235
+ computation_cfg=computation_cfg,
231
236
  )
232
237
  assert tsvd.components_.shape[0] == template_cfg.denoising_rank
233
238
  tsvd = FullProbeTemporalPCAEmbedder.from_sklearn(
@@ -26,7 +26,7 @@ from ..util.job_util import ensure_computation_config
26
26
  from ..util.logging_util import get_logger
27
27
  from ..util.motion import MotionInfo
28
28
  from ..util.noise_util import SpatialWhitener
29
- from ..util.py_util import resolve_path
29
+ from ..util.py_util import ensure_path
30
30
  from ..util.spiketorch import ptp
31
31
  from . import TemplateData, realign
32
32
  from .templib import fit_tsvd, pca_from_templates, quick_mean_templates
@@ -59,7 +59,7 @@ def estimate_template_library(
59
59
  ) -> tuple[DARTsortSorting, TemplateData]:
60
60
  """Postprocess spike train and estimate a TemplateData."""
61
61
  if template_npz_path is not None:
62
- template_npz_path = resolve_path(template_npz_path)
62
+ template_npz_path = ensure_path(template_npz_path)
63
63
  if template_npz_path.exists():
64
64
  return sorting, TemplateData.from_npz(template_npz_path)
65
65
 
@@ -192,7 +192,7 @@ class AmortizedLocalization(BaseWaveformFeaturizer):
192
192
  a0 = masks * pred_amps_alpha1
193
193
  numer = a0.mul(obs_amps).sum(dim=1)
194
194
  denom = a0.square().sum(dim=1)
195
- alphas = numer.div_(denom)
195
+ alphas = numer.div_(denom + 1e-6)
196
196
  if return_pred:
197
197
  return alphas, alphas.unsqueeze(1) * pred_amps_alpha1
198
198
  return alphas
@@ -203,13 +203,13 @@ class AmortizedLocalization(BaseWaveformFeaturizer):
203
203
  if self.localization_model == "gaussian":
204
204
  pred_amps_alpha1 = dists.square().mul(-2).exp()
205
205
  else:
206
- pred_amps_alpha1 = 1.0 / dists
206
+ pred_amps_alpha1 = 1.0 / (dists + 1e-6)
207
207
  alphas, pred_amps = self.get_alphas(
208
208
  obs_amps, pred_amps_alpha1, masks, return_pred=True
209
209
  )
210
210
  else:
211
211
  alphas = F.softplus(z[:, 3])
212
- pred_amps = alphas.unsqueeze(1) / dists
212
+ pred_amps = alphas.unsqueeze(1) / (dists + 1e-6)
213
213
 
214
214
  return alphas, pred_amps
215
215
 
@@ -13,7 +13,7 @@ from ..util.internal_config import (
13
13
  WaveformConfig,
14
14
  )
15
15
  from ..util.logging_util import get_logger
16
- from ..util.py_util import resolve_path
16
+ from ..util.py_util import ensure_path
17
17
  from ..util.waveform_util import assert_all_finite_in_probe
18
18
  from .transform_base import BaseWaveformFeaturizer, BaseWaveformModule
19
19
 
@@ -297,7 +297,7 @@ class WaveformPipeline(torch.nn.Module):
297
297
  del waveforms
298
298
 
299
299
  if hdf5_filename is not None:
300
- hdf5_filename = resolve_path(hdf5_filename, strict=True)
300
+ hdf5_filename = ensure_path(hdf5_filename, strict=True)
301
301
 
302
302
  if self.safe:
303
303
  assert torch.is_tensor(features["waveforms"])
@@ -377,7 +377,7 @@ class WaveformPipeline(torch.nn.Module):
377
377
  if up_to_index == 0:
378
378
  return
379
379
 
380
- hdf5_filename = resolve_path(hdf5_filename, strict=True)
380
+ hdf5_filename = ensure_path(hdf5_filename, strict=True)
381
381
  dev = self.device
382
382
 
383
383
  # use sorting as a way to load all 1d features, which transformers
@@ -139,7 +139,7 @@ class TemplateWaveformReducer(BaseWaveformFeaturizer):
139
139
  dev = computation_cfg.actual_device()
140
140
 
141
141
  n_jobs, Executor, context, *_ = pool_from_cfg(
142
- computation_cfg, check_local=True, small=True
142
+ computation_cfg, check_local=True, small=True, cpu=dev.type == 'cpu'
143
143
  )
144
144
  with Executor(
145
145
  max_workers=n_jobs,