DARTsort 0.5.12__tar.gz → 0.5.13__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. {dartsort-0.5.12 → dartsort-0.5.13}/PKG-INFO +1 -1
  2. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/agglomerate.py +2 -7
  3. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/clustering.py +22 -3
  4. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/clustering_features.py +70 -22
  5. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/mixture.py +1 -1
  6. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/analysis.py +8 -1
  7. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/comparison.py +2 -2
  8. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/grab.py +2 -0
  9. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching.py +13 -1
  10. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/peel_base.py +2 -0
  11. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/amortized_localization.py +3 -3
  12. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/mixture_classifier.py +102 -72
  13. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/temporal_pca.py +15 -3
  14. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/data_util.py +25 -4
  15. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/internal_config.py +2 -1
  16. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/noise_util.py +18 -2
  17. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/analysis_plots.py +99 -6
  18. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/mixture.py +28 -7
  19. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/scatterplots.py +1 -1
  20. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/vismain.py +2 -0
  21. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/PKG-INFO +1 -1
  22. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/SOURCES.txt +1 -52
  23. dartsort-0.5.13/tests/test_agglomerate.py +35 -0
  24. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_data_util.py +1 -0
  25. dartsort-0.5.13/tests/test_mixture_classifier.py +138 -0
  26. dartsort-0.5.12/notebook/basic_integration_test.py +0 -137
  27. dartsort-0.5.12/notebook/ci diag.py +0 -573
  28. dartsort-0.5.12/notebook/dartsort_dredge_demo.ipynb +0 -437
  29. dartsort-0.5.12/notebook/dartsort_dredge_demo.py +0 -144
  30. dartsort-0.5.12/notebook/data test.py +0 -259
  31. dartsort-0.5.12/notebook/dead reckoning.py +0 -361
  32. dartsort-0.5.12/notebook/denoise np1.py +0 -356
  33. dartsort-0.5.12/notebook/denoise np2.py +0 -480
  34. dartsort-0.5.12/notebook/exploratory.py +0 -431
  35. dartsort-0.5.12/notebook/geom debug.py +0 -167
  36. dartsort-0.5.12/notebook/hdbscan_clustering_5min.py +0 -1294
  37. dartsort-0.5.12/notebook/hybrid 10.py +0 -1662
  38. dartsort-0.5.12/notebook/hybrid cluster.py +0 -475
  39. dartsort-0.5.12/notebook/hybrid_11.py +0 -1214
  40. dartsort-0.5.12/notebook/initial training.py +0 -162
  41. dartsort-0.5.12/notebook/make hybrid.py +0 -1376
  42. dartsort-0.5.12/notebook/mgh.py +0 -514
  43. dartsort-0.5.12/notebook/np1 feats.py +0 -1211
  44. dartsort-0.5.12/notebook/np2 processing version c.py +0 -321
  45. dartsort-0.5.12/notebook/paperfigs.py +0 -907
  46. dartsort-0.5.12/notebook/pdists.py +0 -641
  47. dartsort-0.5.12/notebook/postcheck.py +0 -803
  48. dartsort-0.5.12/notebook/ptp vae.py +0 -206
  49. dartsort-0.5.12/notebook/re clust.py +0 -893
  50. dartsort-0.5.12/notebook/reg fig loop.py +0 -723
  51. dartsort-0.5.12/notebook/reg_sim_exp.py +0 -493
  52. dartsort-0.5.12/notebook/relocating.py +0 -284
  53. dartsort-0.5.12/notebook/relocating_all.py +0 -182
  54. dartsort-0.5.12/notebook/restats +0 -0
  55. dartsort-0.5.12/notebook/sortStinger1.py +0 -1590
  56. dartsort-0.5.12/notebook/sortStinger3.py +0 -1541
  57. dartsort-0.5.12/notebook/sortStinger4.py +0 -1835
  58. dartsort-0.5.12/notebook/sortStinger5.py +0 -1586
  59. dartsort-0.5.12/notebook/sortStinger6.py +0 -1915
  60. dartsort-0.5.12/notebook/spike farm.py +0 -100
  61. dartsort-0.5.12/notebook/ss_ensemble_sim.py +0 -1965
  62. dartsort-0.5.12/notebook/ss_ensemble_sim_drift.py +0 -1928
  63. dartsort-0.5.12/notebook/ss_sim.py +0 -1965
  64. dartsort-0.5.12/notebook/stan diagnostics.py +0 -207
  65. dartsort-0.5.12/notebook/storytelling c.py +0 -686
  66. dartsort-0.5.12/notebook/storytelling.py +0 -684
  67. dartsort-0.5.12/notebook/subtract_with_phaseshift_denoiser.py +0 -260
  68. dartsort-0.5.12/notebook/tests/debug_matching.py +0 -454
  69. dartsort-0.5.12/notebook/tests/test_agreement.py +0 -367
  70. dartsort-0.5.12/notebook/tests/test_amplitude_scaling.py +0 -737
  71. dartsort-0.5.12/notebook/tests/test_batch_optim.py +0 -679
  72. dartsort-0.5.12/notebook/tests/test_ibme.py +0 -86
  73. dartsort-0.5.12/notebook/tests/test_mm_initialization.py +0 -142
  74. dartsort-0.5.12/notebook/tests/test_normxcorr1d.py +0 -63
  75. dartsort-0.5.12/notebook/tests/test_reassignment.py +0 -422
  76. dartsort-0.5.12/notebook/tests/test_relocation.py +0 -230
  77. dartsort-0.5.12/notebook/thomas.py +0 -1004
  78. dartsort-0.5.12/tests/test_mixture_classifier.py +0 -65
  79. {dartsort-0.5.12 → dartsort-0.5.13}/.github/workflows/ci.yml +0 -0
  80. {dartsort-0.5.12 → dartsort-0.5.13}/.github/workflows/ci_windows.yml +0 -0
  81. {dartsort-0.5.12 → dartsort-0.5.13}/.github/workflows/deploy.yml +0 -0
  82. {dartsort-0.5.12 → dartsort-0.5.13}/.github/workflows/docs.yml +0 -0
  83. {dartsort-0.5.12 → dartsort-0.5.13}/.gitignore +0 -0
  84. {dartsort-0.5.12 → dartsort-0.5.13}/README.md +0 -0
  85. {dartsort-0.5.12 → dartsort-0.5.13}/docs/index.md +0 -0
  86. {dartsort-0.5.12 → dartsort-0.5.13}/docs/main_api.md +0 -0
  87. {dartsort-0.5.12 → dartsort-0.5.13}/docs/toolkit_api.md +0 -0
  88. {dartsort-0.5.12 → dartsort-0.5.13}/environment.yml +0 -0
  89. {dartsort-0.5.12 → dartsort-0.5.13}/notebook/.keep +0 -0
  90. {dartsort-0.5.12 → dartsort-0.5.13}/pyproject.toml +0 -0
  91. {dartsort-0.5.12 → dartsort-0.5.13}/setup.cfg +0 -0
  92. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/__init__.py +0 -0
  93. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/cli.py +0 -0
  94. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/__init__.py +0 -0
  95. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/cluster_util.py +0 -0
  96. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/density.py +0 -0
  97. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/forward_backward.py +0 -0
  98. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/kmeans.py +0 -0
  99. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/merge.py +0 -0
  100. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/modes.py +0 -0
  101. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/refine_util.py +0 -0
  102. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/config.py +0 -0
  103. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/detect/__init__.py +0 -0
  104. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/detect/detect.py +0 -0
  105. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/detect/detect_filters.py +0 -0
  106. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/__init__.py +0 -0
  107. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/hybrid_util.py +0 -0
  108. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
  109. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/sim_template_tools.py +0 -0
  110. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/simkit.py +0 -0
  111. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/simlib.py +0 -0
  112. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/localize/__init__.py +0 -0
  113. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/localize/localize_torch.py +0 -0
  114. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/localize/localize_util.py +0 -0
  115. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/main.py +0 -0
  116. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/__init__.py +0 -0
  117. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/__init__.py +0 -0
  118. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
  119. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/drifty.py +0 -0
  120. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/matching_base.py +0 -0
  121. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/pairwise.py +0 -0
  122. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
  123. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/peel_lib.py +0 -0
  124. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/reduction_template.py +0 -0
  125. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/shaver.py +0 -0
  126. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/subtract.py +0 -0
  127. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/threshold.py +0 -0
  128. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  129. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/detect_np1.pt +0 -0
  130. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/detect_np2.pt +0 -0
  131. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/glasbey1024.npz +0 -0
  132. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  133. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  134. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  135. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  136. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  137. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/__init__.py +0 -0
  138. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/get_templates.py +0 -0
  139. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/postprocess_util.py +0 -0
  140. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/realignment.py +0 -0
  141. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/template_util.py +0 -0
  142. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/templates.py +0 -0
  143. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/templib.py +0 -0
  144. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/__init__.py +0 -0
  145. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
  146. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/all_transformers.py +0 -0
  147. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/amplitudes.py +0 -0
  148. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/decollider.py +0 -0
  149. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/denoising_scorer.py +0 -0
  150. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/enforce_decrease.py +0 -0
  151. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/fixed_prop.py +0 -0
  152. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/interp.py +0 -0
  153. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/localize.py +0 -0
  154. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/matching_denoiser.py +0 -0
  155. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/pipeline.py +0 -0
  156. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/reduction.py +0 -0
  157. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/single_channel_denoiser.py +0 -0
  158. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/supervised_denoiser.py +0 -0
  159. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/transform_base.py +2 -2
  160. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/whiten.py +0 -0
  161. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/__init__.py +0 -0
  162. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/cli_util.py +0 -0
  163. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/drift_util.py +0 -0
  164. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/interpolation_util.py +0 -0
  165. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/job_util.py +0 -0
  166. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/logging_util.py +0 -0
  167. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/main_util.py +0 -0
  168. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/more_operators.py +0 -0
  169. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/motion.py +0 -0
  170. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/multiprocessing_util.py +0 -0
  171. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/nn_util.py +0 -0
  172. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/peel_util.py +0 -0
  173. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/preprocess_util.py +0 -0
  174. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/py_util.py +0 -0
  175. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/registration_util.py +0 -0
  176. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/residual_util.py +0 -0
  177. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/sparse_util.py +0 -0
  178. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/spikeio.py +0 -0
  179. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/spiketorch.py +0 -0
  180. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/testing_util/__init__.py +0 -0
  181. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
  182. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
  183. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/torch_optimization_util.py +0 -0
  184. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/torch_util.py +0 -0
  185. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/waveform_util.py +0 -0
  186. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/__init__.py +0 -0
  187. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/colors.py +0 -0
  188. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/gt.py +0 -0
  189. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/layout.py +0 -0
  190. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/metrics_util.py +0 -0
  191. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/recanim.py +0 -0
  192. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/sorting.py +0 -0
  193. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/unit.py +0 -0
  194. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/unit_comparison.py +0 -0
  195. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/versus.py +0 -0
  196. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/waveforms.py +0 -0
  197. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/dependency_links.txt +0 -0
  198. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/entry_points.txt +0 -0
  199. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/requires.txt +0 -0
  200. {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/top_level.txt +0 -0
  201. {dartsort-0.5.12 → dartsort-0.5.13}/tests/conftest.py +0 -0
  202. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_alignment.py +0 -0
  203. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_cluster_util.py +0 -0
  204. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_clustering.py +0 -0
  205. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_config.py +0 -0
  206. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_dartsort.py +0 -0
  207. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_density.py +0 -0
  208. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_detect.py +0 -0
  209. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_drift_util.py +0 -0
  210. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_drifty_matching.py +0 -0
  211. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_enforce_decrease.py +0 -0
  212. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_grab_and_featurize.py +0 -0
  213. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_kmeans.py +0 -0
  214. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_linalg.py +0 -0
  215. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_localize_torch.py +0 -0
  216. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_matching.py +0 -0
  217. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_modeling.py +0 -0
  218. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_simkit.py +0 -0
  219. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_spiketorch.py +0 -0
  220. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_subtract.py +0 -0
  221. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_templates.py +0 -0
  222. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_threshold.py +0 -0
  223. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_torch_optimization_util.py +0 -0
  224. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_transform.py +0 -0
  225. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_util.py +0 -0
  226. {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_waveform_util.py +0 -0
  227. {dartsort-0.5.12 → dartsort-0.5.13}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dartsort
3
- Version: 0.5.12
3
+ Version: 0.5.13
4
4
  Summary: DARTsort
5
5
  Author-email: Charlie Windolf <ciw2048@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/cwindolf/dartsort
@@ -770,7 +770,7 @@ def combine_gmm_scores(
770
770
  **{
771
771
  f"{new_prefix}_candidates": candidates,
772
772
  f"{new_prefix}_responsibilities": responsibilities,
773
- f"{new_prefix}_logliks": logliks,
773
+ f"{new_prefix}_log_liks": logliks,
774
774
  }
775
775
  )
776
776
 
@@ -826,7 +826,7 @@ def combine_gmm_scores(
826
826
  **{
827
827
  f"{new_prefix}_candidates": cand,
828
828
  f"{new_prefix}_responsibilities": mergedr,
829
- f"{new_prefix}_logliks": mergedl,
829
+ f"{new_prefix}_log_liks": mergedl,
830
830
  },
831
831
  )
832
832
 
@@ -873,11 +873,6 @@ def deduplicate_spikes(
873
873
  radius_ms: float = -1.0,
874
874
  score_by=("merged_log_liks", "gmm_log_liks", "scores"),
875
875
  ) -> DARTsortSorting:
876
- """The lower-scoring of any spikes within radius_samples of each other is relabeled to -1.
877
-
878
- scores are grabbed from the first entry of score_by which is a valid property of the sorting.
879
- If it's mu
880
- """
881
876
  if radius_ms < 0 or sorting.labels is None:
882
877
  return sorting
883
878
 
@@ -155,10 +155,23 @@ class Clusterer:
155
155
  def handle_sampling(
156
156
  self, features: SimpleMatrixFeatures
157
157
  ) -> tuple[Literal[False], slice] | tuple[Literal[True], np.ndarray]:
158
+ if features.keep_mask is not None:
159
+ mask_ix = np.flatnonzero(features.keep_mask)
160
+ else:
161
+ mask_ix = None
158
162
  if self.sampling_cfg is None:
159
- return False, slice(None)
163
+ if mask_ix is not None:
164
+ return True, mask_ix
165
+ else:
166
+ return False, slice(None)
160
167
  elif features.n <= self.sampling_cfg.n_waveforms_fit:
161
- return False, slice(None)
168
+ if mask_ix is not None:
169
+ return True, mask_ix
170
+ else:
171
+ return False, slice(None)
172
+
173
+ if mask_ix is not None:
174
+ features = features.mask(mask_ix)
162
175
 
163
176
  weights = fit_reweighting(
164
177
  voltages=features.signed_amplitudes,
@@ -173,6 +186,8 @@ class Clusterer:
173
186
  replace=False,
174
187
  )
175
188
  ixs.sort()
189
+ if mask_ix is not None:
190
+ ixs = mask_ix[ixs]
176
191
  return True, ixs
177
192
 
178
193
  def cluster(
@@ -432,16 +447,20 @@ class DensityPeaksClusterer(Clusterer):
432
447
  kdtree = res["kdtree"]
433
448
  assert isinstance(ixs, np.ndarray)
434
449
  rest = np.setdiff1d(np.arange(len(X)), ixs)
450
+ Xr = X[rest]
451
+ isna = np.isnan(Xr).any(axis=1)
452
+ np.nan_to_num(Xr, copy=False)
435
453
  other_labels = density.nearest_neighbor_assign(
436
454
  kdtree,
437
455
  res["labels"],
438
- X[rest],
456
+ Xr,
439
457
  radius_search=self.radius_search,
440
458
  workers=self.workers,
441
459
  )
442
460
  labels = cluster_util.combine_disjoint(
443
461
  ixs, res["labels"], rest, other_labels
444
462
  )
463
+ labels[rest[isna]] = -1
445
464
  else:
446
465
  kdtree = None
447
466
  labels = res["labels"]
@@ -1,4 +1,5 @@
1
1
  from typing import Self, cast
2
+ from logging import WARN
2
3
 
3
4
  import h5py
4
5
  import numpy as np
@@ -38,9 +39,14 @@ class SimpleMatrixFeatures:
38
39
  xyza: np.ndarray | None
39
40
  signed_amplitudes: np.ndarray
40
41
  amplitudes: np.ndarray
42
+ keep_mask: np.ndarray | None
41
43
 
42
44
  def mask(self, mask) -> Self:
43
45
  a = self.amplitudes[mask]
46
+ if self.keep_mask is not None:
47
+ k = self.keep_mask[mask]
48
+ else:
49
+ k = None
44
50
  return self.__class__(
45
51
  n=a.shape[0],
46
52
  features=self.features[mask],
@@ -50,6 +56,7 @@ class SimpleMatrixFeatures:
50
56
  xyza=None if self.xyza is None else self.xyza[mask],
51
57
  signed_amplitudes=self.signed_amplitudes[mask],
52
58
  amplitudes=a,
59
+ keep_mask=k,
53
60
  )
54
61
 
55
62
  @classmethod
@@ -68,14 +75,19 @@ class SimpleMatrixFeatures:
68
75
  )
69
76
  if xyza is not None:
70
77
  x = xyza[:, 0]
71
- if not _allfinite(x):
72
- raise ValueError(_numbers_error_str("x", x))
78
+ _check_numbers(
79
+ "x", x, raise_for_numerics=clustering_features_cfg.raise_for_numerics
80
+ )
73
81
  z = xyza[:, 2]
74
- if not _allfinite(z):
75
- raise ValueError(_numbers_error_str("z", z))
82
+ _check_numbers(
83
+ "z", z, raise_for_numerics=clustering_features_cfg.raise_for_numerics
84
+ )
76
85
  z_reg = motion.correct_s(t_s, z)
77
- if not _allfinite(z_reg):
78
- raise ValueError(_numbers_error_str("z_reg", z_reg))
86
+ _check_numbers(
87
+ "z_reg",
88
+ z_reg,
89
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
90
+ )
79
91
  else:
80
92
  x = z = z_reg = None
81
93
 
@@ -96,14 +108,20 @@ class SimpleMatrixFeatures:
96
108
  amp = getattr(sorting, clustering_features_cfg.amplitudes_dataset_name)
97
109
  if clustering_features_cfg.use_amplitude:
98
110
  assert amp is not None
99
- if not _allfinite(amp):
100
- raise ValueError(_numbers_error_str("amp", amp))
111
+ _check_numbers(
112
+ "amp",
113
+ amp,
114
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
115
+ )
101
116
  ampft = amp.copy()
102
117
  if clustering_features_cfg.log_transform_amplitude:
103
118
  ampft = np.log(clustering_features_cfg.amp_log_c + ampft)
104
119
  ampft *= clustering_features_cfg.amp_scale
105
- if not _allfinite(ampft):
106
- raise ValueError(_numbers_error_str("ampft", ampft))
120
+ _check_numbers(
121
+ "ampft",
122
+ ampft,
123
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
124
+ )
107
125
  features.append(ampft[:, None])
108
126
 
109
127
  v = getattr(sorting, clustering_features_cfg.voltages_dataset_name, None)
@@ -124,8 +142,11 @@ class SimpleMatrixFeatures:
124
142
  rank=clustering_features_cfg.n_main_channel_pcs,
125
143
  dataset_name=clustering_features_cfg.pca_dataset_name,
126
144
  )
127
- if not _allfinite(pcs):
128
- raise ValueError(_numbers_error_str("No motion pcs", pcs))
145
+ _check_numbers(
146
+ "No motion pcs",
147
+ pcs,
148
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
149
+ )
129
150
  elif do_pcs and clustering_features_cfg.motion_aware:
130
151
  shifts, n_pitches_shift = motion.pitch_shifts(
131
152
  sorting=sorting,
@@ -144,8 +165,11 @@ class SimpleMatrixFeatures:
144
165
  mask = np.broadcast_to(mask, len(schan))
145
166
  if hasattr(sorting, clustering_features_cfg.pca_dataset_name):
146
167
  pcs = getattr(sorting, clustering_features_cfg.pca_dataset_name)
147
- if not _allfinite(pcs):
148
- raise ValueError(_numbers_error_str("sorting pcs", pcs))
168
+ _check_numbers(
169
+ "sorting pcs",
170
+ pcs,
171
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
172
+ )
149
173
  erp, pcs = interpolate_by_chunk(
150
174
  mask=mask,
151
175
  dataset=pcs,
@@ -158,8 +182,11 @@ class SimpleMatrixFeatures:
158
182
  params=clustering_features_cfg.interp_params,
159
183
  )
160
184
  pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
161
- if not _allfinite(pcs):
162
- raise ValueError(_numbers_error_str("sorting interp pcs", pcs))
185
+ _check_numbers(
186
+ "sorting interp pcs",
187
+ pcs,
188
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
189
+ )
163
190
  else:
164
191
  assert sorting.parent_h5_path is not None
165
192
  with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
@@ -175,8 +202,11 @@ class SimpleMatrixFeatures:
175
202
  params=clustering_features_cfg.interp_params,
176
203
  )
177
204
  pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
178
- if not _allfinite(pcs):
179
- raise ValueError(_numbers_error_str("h5 interp pcs", pcs))
205
+ _check_numbers(
206
+ "h5 interp pcs",
207
+ pcs,
208
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
209
+ )
180
210
 
181
211
  if do_pcs:
182
212
  assert pcs is not None
@@ -192,8 +222,11 @@ class SimpleMatrixFeatures:
192
222
  else:
193
223
  assert pctf in ("none", None)
194
224
  pcs *= clustering_features_cfg.pc_scale
195
- if not _allfinite(pcs):
196
- raise ValueError(_numbers_error_str(f"{pctf} pcs", pcs))
225
+ _check_numbers(
226
+ f"{pctf} pcs",
227
+ pcs,
228
+ raise_for_numerics=clustering_features_cfg.raise_for_numerics,
229
+ )
197
230
  if torch.is_tensor(pcs):
198
231
  pcs = pcs.numpy(force=True)
199
232
  features.append(pcs)
@@ -203,6 +236,9 @@ class SimpleMatrixFeatures:
203
236
  features = np.concatenate(features, axis=1)
204
237
  else:
205
238
  features = np.empty((n, 0))
239
+ keep = np.atleast_1d(np.isfinite(features).all(axis=1))
240
+ if keep.all():
241
+ keep = None
206
242
  return cls(
207
243
  n=n,
208
244
  features=features,
@@ -212,6 +248,7 @@ class SimpleMatrixFeatures:
212
248
  xyza=xyza,
213
249
  amplitudes=amp,
214
250
  signed_amplitudes=samp,
251
+ keep_mask=keep,
215
252
  )
216
253
 
217
254
 
@@ -331,7 +368,18 @@ def _allfinite(x):
331
368
  return np.isfinite(x).all()
332
369
 
333
370
 
334
- def _numbers_error_str(name: str, x: np.ndarray):
371
+ def _check_numbers(name: str, x: np.ndarray, raise_for_numerics=False):
335
372
  if isinstance(x, torch.Tensor):
336
373
  x = x.numpy(force=True)
337
- return f"{name}: {np.isposinf(x).sum()} +inf, {np.isneginf(x).sum()} -inf, {np.isnan(x).sum()} nan."
374
+
375
+ npinf = np.isposinf(x).sum()
376
+ nninf = np.isneginf(x).sum()
377
+ nna = np.isnan(x).sum()
378
+ if not (npinf + nninf + nna):
379
+ return
380
+
381
+ err = f"{name}: {npinf} +inf, {nninf} -inf, {nna} nan."
382
+ if raise_for_numerics:
383
+ raise ValueError(err)
384
+ else:
385
+ logger.warn(err)
@@ -4381,7 +4381,7 @@ def get_full_neighborhood_data(
4381
4381
  assert xfull.shape[2] == full_neighborhoods.b.neighborhoods.shape[1]
4382
4382
  frank = xfull.shape[1]
4383
4383
  if clustering_features_cfg is not None:
4384
- assert frank == clustering_features_cfg.feature_rank
4384
+ assert 0 < frank <= clustering_features_cfg.feature_rank
4385
4385
  assert xfull.shape[0] == full_neighborhoods.b.neighborhood_ids.shape[0]
4386
4386
  xfull = xfull.view(len(xfull), -1)
4387
4387
  xfull = xfull.nan_to_num_()
@@ -108,6 +108,8 @@ class DARTsortAnalysis:
108
108
  "localizations",
109
109
  "positions",
110
110
  ),
111
+ compute_extensions=("correlograms",),
112
+ estimate_si_sparsity=False,
111
113
  try_backup_amplitudes=("amplitudes", "ptp_amplitudes"),
112
114
  quality_metrics=(
113
115
  "sliding_rp_violation",
@@ -288,6 +290,8 @@ class DARTsortAnalysis:
288
290
  motion=motion,
289
291
  features_cfg=clustering_features_cfg,
290
292
  compute_extensions_if_templates=None,
293
+ compute_extensions=compute_extensions,
294
+ estimate_si_sparsity=estimate_si_sparsity,
291
295
  )
292
296
  analyzer.compute("quality_metrics", metric_names=quality_metrics)
293
297
  qc_df = analyzer.get_extension("quality_metrics").get_data()
@@ -379,7 +383,10 @@ class DARTsortAnalysis:
379
383
  return self._summary_df
380
384
  df = self.quality_df.copy()
381
385
  uids = self.sorting.unit_ids
382
- df["ds_unit_amplitudes"] = pd.Series(index=uids, data=self.unit_amplitudes())
386
+ if self.template_data is not None:
387
+ df["ds_unit_amplitudes"] = pd.Series(
388
+ index=uids, data=self.unit_amplitudes()
389
+ )
383
390
  df["ds_firing_rates"] = pd.Series(index=uids, data=self.firing_rates())
384
391
  self._summary_df = df
385
392
  return df
@@ -135,12 +135,12 @@ class DARTsortGroundTruthComparison:
135
135
  return res
136
136
 
137
137
  def _unit_info_dataframe(self, force_distances=False, perf_only=False):
138
- amplitudes = self.gt_analysis.unit_amplitudes()
139
138
  firing_rates = self.gt_analysis.firing_rates()
140
139
  df = self.comparison.get_performance()
141
140
  assert isinstance(df, pd.DataFrame)
142
141
  df = df.astype(float) # not sure what the problem was...
143
- df["gt_ptp_amplitude"] = amplitudes
142
+ if self.gt_analysis.template_data is not None:
143
+ df["gt_ptp_amplitude"] = self.gt_analysis.unit_amplitudes()
144
144
  df["gt_firing_rate"] = firing_rates
145
145
  df["gt_sorter_name"] = self.gt_analysis.name or ""
146
146
  df["tested_sorter_name"] = self.tested_analysis.name or ""
@@ -88,8 +88,10 @@ class GrabAndFeaturize(BasePeeler):
88
88
  skip_features: bool = False,
89
89
  n_resid_snips: int | None = None,
90
90
  to_cpu: bool = True,
91
+ **peel_kw,
91
92
  ) -> "PeelingBatchResult":
92
93
  """Override process_chunk to skip empties."""
94
+ assert not peel_kw
93
95
  if chunk_end_samples is None:
94
96
  chunk_end_samples = min(
95
97
  self.recording.get_num_samples(),
@@ -24,7 +24,7 @@ from ..util.internal_config import (
24
24
  )
25
25
  from ..util.logging_util import get_logger
26
26
  from ..util.motion import MotionInfo
27
- from ..util.waveform_util import make_channel_index
27
+ from ..util.waveform_util import full_channel_index, make_channel_index
28
28
  from .matching_util import (
29
29
  ChunkTemplateData,
30
30
  MatchingPeaks,
@@ -260,6 +260,7 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
260
260
  return_residual=False,
261
261
  return_waveforms=True,
262
262
  return_conv=False,
263
+ return_clean_waveforms=False,
263
264
  ) -> PeelingBatchResult:
264
265
  assert self.matching_templates is not None
265
266
  # get chunk center time and template info at that time
@@ -288,6 +289,7 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
288
289
  return_residual=return_residual,
289
290
  return_conv=return_conv,
290
291
  return_collisioncleaned_waveforms=return_waveforms,
292
+ return_clean_waveforms=return_clean_waveforms,
291
293
  )
292
294
 
293
295
  # process spike times and create return result
@@ -304,9 +306,11 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
304
306
  self,
305
307
  traces: Tensor,
306
308
  chunk_template_data: ChunkTemplateData,
309
+ *,
307
310
  left_margin=0,
308
311
  right_margin=0,
309
312
  return_collisioncleaned_waveforms=True,
313
+ return_clean_waveforms=False,
310
314
  return_residual=False,
311
315
  return_conv=False,
312
316
  unit_mask=None,
@@ -475,6 +479,14 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
475
479
  assert self.p.channel_selection == "template"
476
480
  channels = chunk_template_data.main_channels[peaks.template_inds]
477
481
  waveforms = None
482
+ if return_clean_waveforms:
483
+ ci_full = full_channel_index(self.b.channel_index.shape[0], True)
484
+ ci_full = ci_full.to(self.b.channel_index)
485
+ res["clean_waveforms"] = chunk_template_data.get_clean_waveforms(
486
+ peaks=peaks,
487
+ channels=channels,
488
+ channel_index=ci_full,
489
+ )
478
490
  res["channels"] = channels
479
491
  if return_collisioncleaned_waveforms:
480
492
  assert waveforms is not None
@@ -496,6 +496,7 @@ class BasePeeler(BModule):
496
496
  skip_features: bool = False,
497
497
  n_resid_snips: int | None = None,
498
498
  to_cpu: bool = True,
499
+ **peel_kw,
499
500
  ) -> "PeelingBatchResult":
500
501
  """Grab, peel, and featurize a chunk, returning a dict of numpy arrays
501
502
 
@@ -512,6 +513,7 @@ class BasePeeler(BModule):
512
513
  right_margin=right_margin,
513
514
  return_waveforms=return_waveforms,
514
515
  return_residual=return_residual or bool(n_resid_snips),
516
+ **peel_kw,
515
517
  )
516
518
  chunk_result = self.featurize_chunk_result(
517
519
  peel_result=peel_result,
@@ -162,7 +162,7 @@ class AmortizedLocalization(BaseWaveformFeaturizer):
162
162
  assert obs_amps is not None
163
163
  if neighborhoods is None:
164
164
  neighborhoods = self.b.padded_geom[self.b.model_channel_index[channels]]
165
- w = obs_amps / obs_amps.sum(1, keepdims=True)
165
+ w = obs_amps / obs_amps.sum(1, keepdims=True).clamp(min=1e-6)
166
166
  centers = torch.sum(w.unsqueeze(-1) * neighborhoods, dim=1)
167
167
  return centers
168
168
  else:
@@ -266,9 +266,9 @@ class AmortizedLocalization(BaseWaveformFeaturizer):
266
266
  x_masked = x * mask
267
267
  if self.scale_loss_by_mean:
268
268
  # 1/(n_chans_retained*mean amplitude)
269
- rescale = x_masked.sum(1, keepdim=True).clamp(min=1e-8).reciprocal()
269
+ rescale = x_masked.sum(1, keepdim=True).clamp(min=1e-6).reciprocal()
270
270
  else:
271
- rescale = 1.0 / mask.sum(1, keepdim=True)
271
+ rescale = mask.sum(1, keepdim=True).clamp(min=1e-6).reciprocal()
272
272
  x_masked *= rescale
273
273
  recon_x_masked *= rescale
274
274
  mse = F.mse_loss(recon_x_masked, x_masked, reduction="sum") / self.batch_size
@@ -13,6 +13,7 @@ TODO:
13
13
 
14
14
  import sys
15
15
  from dataclasses import replace
16
+ from math import fabs
16
17
  from typing import TYPE_CHECKING, Sequence
17
18
 
18
19
  import numpy as np
@@ -113,6 +114,11 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
113
114
  self.channel_index_np = self.b.channel_index.numpy(force=True)
114
115
  self.workers = 1
115
116
 
117
+ def get_extra_state(self):
118
+ es = super().get_extra_state()
119
+ es["feature_rank"] = self.feature_rank
120
+ return es
121
+
116
122
  def needs_fit(self):
117
123
  return not hasattr(self, "tmm")
118
124
 
@@ -146,6 +152,48 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
146
152
  cii, cjj = np.nonzero(self.channel_index_np < len(self.channel_index_np))
147
153
  self.channel_index_valid_inds = cii, cjj
148
154
 
155
+ # precompute drifting neighborhood lookup tables
156
+ if self.motion.drifting:
157
+ # in this case, prebake a lookup table
158
+ # TODO just consider relevant time bins here?
159
+ # or chunk this somehow?
160
+ tbs = self.motion.time_bins_s
161
+ tcm_t, nid_t = neighborhood_mapping_at_time(
162
+ motion=self.motion,
163
+ t_s=torch.from_numpy(tbs),
164
+ neighborhoods=self.b.neighborhoods,
165
+ channel_index=self.channel_index_np,
166
+ workers=self.workers,
167
+ static_neighbs=self.static_neighbs,
168
+ channel_index_valid_inds=self.channel_index_valid_inds,
169
+ )
170
+ nid_map = None
171
+ else:
172
+ tcm_t = nid_t = tbs = None
173
+ # in this case, channel index is a superset of neighborhoods and
174
+ # we prebake the mapping from chans to neighb ids
175
+ ci_eq_neighb = _outer_all_equal(
176
+ self.b.channel_index, self.tmm.neighb_cov.obs_ix
177
+ )
178
+ # some neighborhood entries are identical, but they all get covered
179
+ assert torch.all(ci_eq_neighb.sum(0) >= 1)
180
+ # each channel index maps to at most one neighb id
181
+ assert torch.all(ci_eq_neighb.sum(1) <= 1)
182
+ _chans, _chan_nids = ci_eq_neighb.nonzero(as_tuple=True)
183
+ nid_map = _chans.new_full(
184
+ (self.b.channel_index.shape[0],),
185
+ self.tmm.neighb_cov.b.obs_ix.shape[0] + 1,
186
+ )
187
+ nid_map[_chans] = _chan_nids
188
+ self.register_buffer_or_none("neighborhood_ids_map", nid_map)
189
+ self.drifting_time_bins_s = tbs
190
+ self.register_buffer_or_none(
191
+ "drifting_target_channels_map_t", tcm_t, persistent=False
192
+ )
193
+ self.register_buffer_or_none(
194
+ "drifting_neighborhood_ids_map_t", nid_t, persistent=False
195
+ )
196
+
149
197
  def fit(
150
198
  self,
151
199
  recording,
@@ -211,24 +259,6 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
211
259
  assert mix_data.tmm.noise is not None
212
260
  self.feature_rank = mix_data.tmm.noise.rank
213
261
 
214
- if not self.motion.drifting:
215
- # in this case, channel index is a superset of neighborhoods and
216
- # we prebake the mapping from chans to neighb ids
217
- ci_eq_neighb = _outer_all_equal(
218
- self.b.channel_index, mix_data.tmm.neighb_cov.obs_ix
219
- )
220
- # some neighborhood entries are identical, but they all get covered
221
- assert torch.all(ci_eq_neighb.sum(0) >= 1)
222
- # each channel index maps to at most one neighb id
223
- assert torch.all(ci_eq_neighb.sum(1) <= 1)
224
- _chans, _chan_nids = ci_eq_neighb.nonzero(as_tuple=True)
225
- nid_map = _chans.new_full(
226
- (self.b.channel_index.shape[0],),
227
- mix_data.tmm.neighb_cov.b.obs_ix.shape[0] + 1,
228
- )
229
- nid_map[_chans] = _chan_nids
230
- self.register_buffer("neighborhood_ids_map", nid_map)
231
-
232
262
  self.erp: StableFeaturesInterpolator = stable_features.erp
233
263
 
234
264
  # de-puff the lut for space reasons
@@ -268,6 +298,8 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
268
298
 
269
299
  assert hasattr(self, "motion")
270
300
  assert self.motion is not None
301
+ extra_state = state_dict[f"{prefix}_extra_state"]
302
+ self.feature_rank = extra_state["feature_rank"]
271
303
 
272
304
  # reconstruct non-__init__ modules...
273
305
  stripped_dict = {k.removeprefix(prefix): v for k, v in state_dict.items()}
@@ -318,15 +350,10 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
318
350
  # drifting channel mapping
319
351
  assert self.motion is not None
320
352
  if self.motion.drifting:
321
- target_channels_map, neighborhood_ids_map = neighborhood_mapping_at_time(
322
- motion=self.motion,
323
- t_s=chunk_center_s,
324
- neighborhoods=self.b.neighborhoods,
325
- channel_index=self.channel_index_np,
326
- workers=self.workers,
327
- static_neighbs=self.static_neighbs,
328
- channel_index_valid_inds=self.channel_index_valid_inds,
329
- )
353
+ assert self.drifting_time_bins_s is not None
354
+ t_idx = find_nearest(self.drifting_time_bins_s, chunk_center_s)
355
+ target_channels_map = self.b.drifting_target_channels_map_t[t_idx]
356
+ neighborhood_ids_map = self.b.drifting_neighborhood_ids_map_t[t_idx]
330
357
  else:
331
358
  target_channels_map = self.b.channel_index
332
359
  neighborhood_ids_map = self.b.neighborhood_ids_map
@@ -376,62 +403,57 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
376
403
 
377
404
  def neighborhood_mapping_at_time(
378
405
  motion: MotionInfo,
379
- t_s: torch.Tensor,
406
+ *,
407
+ t_s: torch.Tensor | np.ndarray,
380
408
  channel_index: np.ndarray,
381
409
  neighborhoods: torch.Tensor,
382
410
  workers: int = 4,
383
411
  shift_mode="round",
384
- static_neighbs: np.ndarray | None = None,
385
- channel_index_valid_inds: tuple[np.ndarray, np.ndarray] | None = None,
412
+ static_neighbs: np.ndarray,
413
+ channel_index_valid_inds: tuple[np.ndarray, np.ndarray],
386
414
  ):
387
- # replicates static_channel_neighborhoods
388
-
389
- # shifted geom neighborhoods at time
390
- t_s = t_s.numpy(force=True)
391
- if t_s.size > 1:
392
- t_s = t_s.mean()
393
- probe_disp = -motion.disp_at_s(times_s=t_s, depths_um=motion.geom[:, 1], grid=True)
394
- assert probe_disp.shape == (motion.geom.shape[0], 1)
395
- if shift_mode == "floor":
396
- n_pitches_shift = (probe_disp / motion.pitch).astype(np.int32)
397
- elif shift_mode == "round":
398
- n_pitches_shift = np.round(probe_disp / motion.pitch).astype(np.int32)
399
- else:
400
- assert False
401
-
402
- if channel_index_valid_inds is None:
403
- cii, cjj = np.nonzero(channel_index < len(channel_index))
404
- else:
405
- cii, cjj = channel_index_valid_inds
415
+ t_s = t_s.numpy(force=True) if isinstance(t_s, torch.Tensor) else np.asarray(t_s)
416
+ t_s = np.atleast_1d(t_s)
417
+
418
+ index_at_time = torch.full((len(t_s), *channel_index.shape), -1)
419
+ nids_at_time = torch.full(
420
+ (len(t_s), channel_index.shape[0]),
421
+ neighborhoods.shape[0],
422
+ device=neighborhoods.device,
423
+ )
406
424
 
407
- shift = n_pitches_shift * motion.pitch
408
- if static_neighbs is None:
409
- geomp = np.pad(motion.geom, [(0, 1), (0, 0)], constant_values=np.nan)
410
- shifted_neighbs = geomp[channel_index]
411
- else:
425
+ for j, t in enumerate(t_s):
426
+ probe_disp = -motion.disp_at_s(
427
+ times_s=t, depths_um=motion.geom[:, 1], grid=True
428
+ )
429
+ if shift_mode == "floor":
430
+ n_pitches_shift = (probe_disp / motion.pitch).astype(np.int32)
431
+ elif shift_mode == "round":
432
+ n_pitches_shift = np.round(probe_disp / motion.pitch).astype(np.int32)
433
+ else:
434
+ assert False
435
+ shift = n_pitches_shift * motion.pitch
412
436
  shifted_neighbs = static_neighbs.copy()
413
- shifted_neighbs[:, :, 1] += shift
414
-
415
- # uneighbxz, uinv = np.unique(shifted_neighbs[cii, cjj], axis=0, return_inverse=True)
437
+ shifted_neighbs[:, :, 1] += shift
438
+ _, umatch = motion.rgeom_kdt.query(
439
+ shifted_neighbs[channel_index_valid_inds],
440
+ distance_upper_bound=motion.min_dist,
441
+ workers=workers,
442
+ )
443
+ index = np.full(channel_index.shape, motion.rgeom.shape[0])
444
+ index[channel_index_valid_inds] = umatch
445
+ index = torch.asarray(index, device=neighborhoods.device)
446
+ assert index.shape[1] == neighborhoods.shape[1]
416
447
 
417
- # match shifted geom channels to rgeom channels
418
- _, umatch = motion.rgeom_kdt.query(
419
- shifted_neighbs[cii, cjj], distance_upper_bound=motion.min_dist, workers=workers
420
- )
448
+ index_at_time[j] = index
421
449
 
422
- # get shifted channel neighborhoods
423
- index = np.full(channel_index.shape, motion.rgeom.shape[0])
424
- index[cii, cjj] = umatch
450
+ mapping = _outer_all_equal(index, neighborhoods)
451
+ _chans, _chan_nids = mapping.nonzero(as_tuple=True)
452
+ nids_at_time[j, _chans] = _chan_nids
425
453
 
426
- # mapping between these and `neighborhoods`
427
- index = torch.asarray(index, device=neighborhoods.device)
428
- assert index.shape[1] == neighborhoods.shape[1]
429
- mapping = _outer_all_equal(index, neighborhoods)
430
- _chans, _chan_nids = mapping.nonzero(as_tuple=True)
431
- nid_map = _chans.new_full((channel_index.shape[0],), neighborhoods.shape[0])
432
- nid_map[_chans] = _chan_nids
454
+ assert (index_at_time >= 0).all()
433
455
 
434
- return index, nid_map
456
+ return index_at_time, nids_at_time
435
457
 
436
458
 
437
459
  @torch_compile
@@ -451,3 +473,11 @@ def _outer_all_equal(x: torch.Tensor, y: torch.Tensor):
451
473
  out.logical_and_(torch.eq(x[:, :, j], y[:, :, j], out=msk))
452
474
 
453
475
  return out
476
+
477
+
478
+ def find_nearest(x, v):
479
+ idx = np.searchsorted(x, v, side="left")
480
+ if idx > 0 and (idx == len(x) or fabs(v - x[idx - 1]) < fabs(v - x[idx])):
481
+ return idx - 1
482
+ else:
483
+ return idx