DARTsort 0.5.12__tar.gz → 0.5.14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.12 → dartsort-0.5.14}/PKG-INFO +1 -1
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/agglomerate.py +2 -7
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/clustering.py +22 -3
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/clustering_features.py +70 -22
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/mixture.py +4 -2
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/config.py +3 -3
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/analysis.py +8 -1
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/comparison.py +8 -3
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/grab.py +2 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching.py +13 -1
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/peel_base.py +13 -1
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/peel_lib.py +64 -32
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/_multichan_denoiser_kit.py +109 -34
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/amortized_localization.py +3 -3
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/amplitudes.py +13 -6
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/decollider.py +18 -8
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/mixture_classifier.py +102 -72
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/pipeline.py +12 -6
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/temporal_pca.py +15 -3
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/data_util.py +25 -4
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/internal_config.py +7 -5
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/main_util.py +2 -1
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/noise_util.py +18 -2
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/peel_util.py +16 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/analysis_plots.py +99 -6
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/mixture.py +28 -7
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/scatterplots.py +1 -1
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/vismain.py +2 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/SOURCES.txt +1 -52
- dartsort-0.5.14/tests/test_agglomerate.py +35 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_data_util.py +1 -0
- dartsort-0.5.14/tests/test_mixture_classifier.py +138 -0
- dartsort-0.5.12/notebook/basic_integration_test.py +0 -137
- dartsort-0.5.12/notebook/ci diag.py +0 -573
- dartsort-0.5.12/notebook/dartsort_dredge_demo.ipynb +0 -437
- dartsort-0.5.12/notebook/dartsort_dredge_demo.py +0 -144
- dartsort-0.5.12/notebook/data test.py +0 -259
- dartsort-0.5.12/notebook/dead reckoning.py +0 -361
- dartsort-0.5.12/notebook/denoise np1.py +0 -356
- dartsort-0.5.12/notebook/denoise np2.py +0 -480
- dartsort-0.5.12/notebook/exploratory.py +0 -431
- dartsort-0.5.12/notebook/geom debug.py +0 -167
- dartsort-0.5.12/notebook/hdbscan_clustering_5min.py +0 -1294
- dartsort-0.5.12/notebook/hybrid 10.py +0 -1662
- dartsort-0.5.12/notebook/hybrid cluster.py +0 -475
- dartsort-0.5.12/notebook/hybrid_11.py +0 -1214
- dartsort-0.5.12/notebook/initial training.py +0 -162
- dartsort-0.5.12/notebook/make hybrid.py +0 -1376
- dartsort-0.5.12/notebook/mgh.py +0 -514
- dartsort-0.5.12/notebook/np1 feats.py +0 -1211
- dartsort-0.5.12/notebook/np2 processing version c.py +0 -321
- dartsort-0.5.12/notebook/paperfigs.py +0 -907
- dartsort-0.5.12/notebook/pdists.py +0 -641
- dartsort-0.5.12/notebook/postcheck.py +0 -803
- dartsort-0.5.12/notebook/ptp vae.py +0 -206
- dartsort-0.5.12/notebook/re clust.py +0 -893
- dartsort-0.5.12/notebook/reg fig loop.py +0 -723
- dartsort-0.5.12/notebook/reg_sim_exp.py +0 -493
- dartsort-0.5.12/notebook/relocating.py +0 -284
- dartsort-0.5.12/notebook/relocating_all.py +0 -182
- dartsort-0.5.12/notebook/restats +0 -0
- dartsort-0.5.12/notebook/sortStinger1.py +0 -1590
- dartsort-0.5.12/notebook/sortStinger3.py +0 -1541
- dartsort-0.5.12/notebook/sortStinger4.py +0 -1835
- dartsort-0.5.12/notebook/sortStinger5.py +0 -1586
- dartsort-0.5.12/notebook/sortStinger6.py +0 -1915
- dartsort-0.5.12/notebook/spike farm.py +0 -100
- dartsort-0.5.12/notebook/ss_ensemble_sim.py +0 -1965
- dartsort-0.5.12/notebook/ss_ensemble_sim_drift.py +0 -1928
- dartsort-0.5.12/notebook/ss_sim.py +0 -1965
- dartsort-0.5.12/notebook/stan diagnostics.py +0 -207
- dartsort-0.5.12/notebook/storytelling c.py +0 -686
- dartsort-0.5.12/notebook/storytelling.py +0 -684
- dartsort-0.5.12/notebook/subtract_with_phaseshift_denoiser.py +0 -260
- dartsort-0.5.12/notebook/tests/debug_matching.py +0 -454
- dartsort-0.5.12/notebook/tests/test_agreement.py +0 -367
- dartsort-0.5.12/notebook/tests/test_amplitude_scaling.py +0 -737
- dartsort-0.5.12/notebook/tests/test_batch_optim.py +0 -679
- dartsort-0.5.12/notebook/tests/test_ibme.py +0 -86
- dartsort-0.5.12/notebook/tests/test_mm_initialization.py +0 -142
- dartsort-0.5.12/notebook/tests/test_normxcorr1d.py +0 -63
- dartsort-0.5.12/notebook/tests/test_reassignment.py +0 -422
- dartsort-0.5.12/notebook/tests/test_relocation.py +0 -230
- dartsort-0.5.12/notebook/thomas.py +0 -1004
- dartsort-0.5.12/tests/test_mixture_classifier.py +0 -65
- {dartsort-0.5.12 → dartsort-0.5.14}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/.gitignore +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/README.md +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/docs/index.md +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/docs/main_api.md +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/environment.yml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/notebook/.keep +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/pyproject.toml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/setup.cfg +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/cluster_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/kmeans.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/modes.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/clustering/refine_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/simkit.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/main.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/subtract.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/transform_base.py +2 -2
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/motion.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/spiketorch.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/conftest.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_alignment.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_clustering.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_config.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_density.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_detect.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_linalg.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_matching.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_modeling.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_simkit.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_subtract.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_templates.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_threshold.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_transform.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.14}/zensical.toml +0 -0
|
@@ -770,7 +770,7 @@ def combine_gmm_scores(
|
|
|
770
770
|
**{
|
|
771
771
|
f"{new_prefix}_candidates": candidates,
|
|
772
772
|
f"{new_prefix}_responsibilities": responsibilities,
|
|
773
|
-
f"{new_prefix}
|
|
773
|
+
f"{new_prefix}_log_liks": logliks,
|
|
774
774
|
}
|
|
775
775
|
)
|
|
776
776
|
|
|
@@ -826,7 +826,7 @@ def combine_gmm_scores(
|
|
|
826
826
|
**{
|
|
827
827
|
f"{new_prefix}_candidates": cand,
|
|
828
828
|
f"{new_prefix}_responsibilities": mergedr,
|
|
829
|
-
f"{new_prefix}
|
|
829
|
+
f"{new_prefix}_log_liks": mergedl,
|
|
830
830
|
},
|
|
831
831
|
)
|
|
832
832
|
|
|
@@ -873,11 +873,6 @@ def deduplicate_spikes(
|
|
|
873
873
|
radius_ms: float = -1.0,
|
|
874
874
|
score_by=("merged_log_liks", "gmm_log_liks", "scores"),
|
|
875
875
|
) -> DARTsortSorting:
|
|
876
|
-
"""The lower-scoring of any spikes within radius_samples of each other is relabeled to -1.
|
|
877
|
-
|
|
878
|
-
scores are grabbed from the first entry of score_by which is a valid property of the sorting.
|
|
879
|
-
If it's mu
|
|
880
|
-
"""
|
|
881
876
|
if radius_ms < 0 or sorting.labels is None:
|
|
882
877
|
return sorting
|
|
883
878
|
|
|
@@ -155,10 +155,23 @@ class Clusterer:
|
|
|
155
155
|
def handle_sampling(
|
|
156
156
|
self, features: SimpleMatrixFeatures
|
|
157
157
|
) -> tuple[Literal[False], slice] | tuple[Literal[True], np.ndarray]:
|
|
158
|
+
if features.keep_mask is not None:
|
|
159
|
+
mask_ix = np.flatnonzero(features.keep_mask)
|
|
160
|
+
else:
|
|
161
|
+
mask_ix = None
|
|
158
162
|
if self.sampling_cfg is None:
|
|
159
|
-
|
|
163
|
+
if mask_ix is not None:
|
|
164
|
+
return True, mask_ix
|
|
165
|
+
else:
|
|
166
|
+
return False, slice(None)
|
|
160
167
|
elif features.n <= self.sampling_cfg.n_waveforms_fit:
|
|
161
|
-
|
|
168
|
+
if mask_ix is not None:
|
|
169
|
+
return True, mask_ix
|
|
170
|
+
else:
|
|
171
|
+
return False, slice(None)
|
|
172
|
+
|
|
173
|
+
if mask_ix is not None:
|
|
174
|
+
features = features.mask(mask_ix)
|
|
162
175
|
|
|
163
176
|
weights = fit_reweighting(
|
|
164
177
|
voltages=features.signed_amplitudes,
|
|
@@ -173,6 +186,8 @@ class Clusterer:
|
|
|
173
186
|
replace=False,
|
|
174
187
|
)
|
|
175
188
|
ixs.sort()
|
|
189
|
+
if mask_ix is not None:
|
|
190
|
+
ixs = mask_ix[ixs]
|
|
176
191
|
return True, ixs
|
|
177
192
|
|
|
178
193
|
def cluster(
|
|
@@ -432,16 +447,20 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
432
447
|
kdtree = res["kdtree"]
|
|
433
448
|
assert isinstance(ixs, np.ndarray)
|
|
434
449
|
rest = np.setdiff1d(np.arange(len(X)), ixs)
|
|
450
|
+
Xr = X[rest]
|
|
451
|
+
isna = np.isnan(Xr).any(axis=1)
|
|
452
|
+
np.nan_to_num(Xr, copy=False)
|
|
435
453
|
other_labels = density.nearest_neighbor_assign(
|
|
436
454
|
kdtree,
|
|
437
455
|
res["labels"],
|
|
438
|
-
|
|
456
|
+
Xr,
|
|
439
457
|
radius_search=self.radius_search,
|
|
440
458
|
workers=self.workers,
|
|
441
459
|
)
|
|
442
460
|
labels = cluster_util.combine_disjoint(
|
|
443
461
|
ixs, res["labels"], rest, other_labels
|
|
444
462
|
)
|
|
463
|
+
labels[rest[isna]] = -1
|
|
445
464
|
else:
|
|
446
465
|
kdtree = None
|
|
447
466
|
labels = res["labels"]
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from typing import Self, cast
|
|
2
|
+
from logging import WARN
|
|
2
3
|
|
|
3
4
|
import h5py
|
|
4
5
|
import numpy as np
|
|
@@ -38,9 +39,14 @@ class SimpleMatrixFeatures:
|
|
|
38
39
|
xyza: np.ndarray | None
|
|
39
40
|
signed_amplitudes: np.ndarray
|
|
40
41
|
amplitudes: np.ndarray
|
|
42
|
+
keep_mask: np.ndarray | None
|
|
41
43
|
|
|
42
44
|
def mask(self, mask) -> Self:
|
|
43
45
|
a = self.amplitudes[mask]
|
|
46
|
+
if self.keep_mask is not None:
|
|
47
|
+
k = self.keep_mask[mask]
|
|
48
|
+
else:
|
|
49
|
+
k = None
|
|
44
50
|
return self.__class__(
|
|
45
51
|
n=a.shape[0],
|
|
46
52
|
features=self.features[mask],
|
|
@@ -50,6 +56,7 @@ class SimpleMatrixFeatures:
|
|
|
50
56
|
xyza=None if self.xyza is None else self.xyza[mask],
|
|
51
57
|
signed_amplitudes=self.signed_amplitudes[mask],
|
|
52
58
|
amplitudes=a,
|
|
59
|
+
keep_mask=k,
|
|
53
60
|
)
|
|
54
61
|
|
|
55
62
|
@classmethod
|
|
@@ -68,14 +75,19 @@ class SimpleMatrixFeatures:
|
|
|
68
75
|
)
|
|
69
76
|
if xyza is not None:
|
|
70
77
|
x = xyza[:, 0]
|
|
71
|
-
|
|
72
|
-
|
|
78
|
+
_check_numbers(
|
|
79
|
+
"x", x, raise_for_numerics=clustering_features_cfg.raise_for_numerics
|
|
80
|
+
)
|
|
73
81
|
z = xyza[:, 2]
|
|
74
|
-
|
|
75
|
-
|
|
82
|
+
_check_numbers(
|
|
83
|
+
"z", z, raise_for_numerics=clustering_features_cfg.raise_for_numerics
|
|
84
|
+
)
|
|
76
85
|
z_reg = motion.correct_s(t_s, z)
|
|
77
|
-
|
|
78
|
-
|
|
86
|
+
_check_numbers(
|
|
87
|
+
"z_reg",
|
|
88
|
+
z_reg,
|
|
89
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
90
|
+
)
|
|
79
91
|
else:
|
|
80
92
|
x = z = z_reg = None
|
|
81
93
|
|
|
@@ -96,14 +108,20 @@ class SimpleMatrixFeatures:
|
|
|
96
108
|
amp = getattr(sorting, clustering_features_cfg.amplitudes_dataset_name)
|
|
97
109
|
if clustering_features_cfg.use_amplitude:
|
|
98
110
|
assert amp is not None
|
|
99
|
-
|
|
100
|
-
|
|
111
|
+
_check_numbers(
|
|
112
|
+
"amp",
|
|
113
|
+
amp,
|
|
114
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
115
|
+
)
|
|
101
116
|
ampft = amp.copy()
|
|
102
117
|
if clustering_features_cfg.log_transform_amplitude:
|
|
103
118
|
ampft = np.log(clustering_features_cfg.amp_log_c + ampft)
|
|
104
119
|
ampft *= clustering_features_cfg.amp_scale
|
|
105
|
-
|
|
106
|
-
|
|
120
|
+
_check_numbers(
|
|
121
|
+
"ampft",
|
|
122
|
+
ampft,
|
|
123
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
124
|
+
)
|
|
107
125
|
features.append(ampft[:, None])
|
|
108
126
|
|
|
109
127
|
v = getattr(sorting, clustering_features_cfg.voltages_dataset_name, None)
|
|
@@ -124,8 +142,11 @@ class SimpleMatrixFeatures:
|
|
|
124
142
|
rank=clustering_features_cfg.n_main_channel_pcs,
|
|
125
143
|
dataset_name=clustering_features_cfg.pca_dataset_name,
|
|
126
144
|
)
|
|
127
|
-
|
|
128
|
-
|
|
145
|
+
_check_numbers(
|
|
146
|
+
"No motion pcs",
|
|
147
|
+
pcs,
|
|
148
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
149
|
+
)
|
|
129
150
|
elif do_pcs and clustering_features_cfg.motion_aware:
|
|
130
151
|
shifts, n_pitches_shift = motion.pitch_shifts(
|
|
131
152
|
sorting=sorting,
|
|
@@ -144,8 +165,11 @@ class SimpleMatrixFeatures:
|
|
|
144
165
|
mask = np.broadcast_to(mask, len(schan))
|
|
145
166
|
if hasattr(sorting, clustering_features_cfg.pca_dataset_name):
|
|
146
167
|
pcs = getattr(sorting, clustering_features_cfg.pca_dataset_name)
|
|
147
|
-
|
|
148
|
-
|
|
168
|
+
_check_numbers(
|
|
169
|
+
"sorting pcs",
|
|
170
|
+
pcs,
|
|
171
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
172
|
+
)
|
|
149
173
|
erp, pcs = interpolate_by_chunk(
|
|
150
174
|
mask=mask,
|
|
151
175
|
dataset=pcs,
|
|
@@ -158,8 +182,11 @@ class SimpleMatrixFeatures:
|
|
|
158
182
|
params=clustering_features_cfg.interp_params,
|
|
159
183
|
)
|
|
160
184
|
pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
|
|
161
|
-
|
|
162
|
-
|
|
185
|
+
_check_numbers(
|
|
186
|
+
"sorting interp pcs",
|
|
187
|
+
pcs,
|
|
188
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
189
|
+
)
|
|
163
190
|
else:
|
|
164
191
|
assert sorting.parent_h5_path is not None
|
|
165
192
|
with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
|
|
@@ -175,8 +202,11 @@ class SimpleMatrixFeatures:
|
|
|
175
202
|
params=clustering_features_cfg.interp_params,
|
|
176
203
|
)
|
|
177
204
|
pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
|
|
178
|
-
|
|
179
|
-
|
|
205
|
+
_check_numbers(
|
|
206
|
+
"h5 interp pcs",
|
|
207
|
+
pcs,
|
|
208
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
209
|
+
)
|
|
180
210
|
|
|
181
211
|
if do_pcs:
|
|
182
212
|
assert pcs is not None
|
|
@@ -192,8 +222,11 @@ class SimpleMatrixFeatures:
|
|
|
192
222
|
else:
|
|
193
223
|
assert pctf in ("none", None)
|
|
194
224
|
pcs *= clustering_features_cfg.pc_scale
|
|
195
|
-
|
|
196
|
-
|
|
225
|
+
_check_numbers(
|
|
226
|
+
f"{pctf} pcs",
|
|
227
|
+
pcs,
|
|
228
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
229
|
+
)
|
|
197
230
|
if torch.is_tensor(pcs):
|
|
198
231
|
pcs = pcs.numpy(force=True)
|
|
199
232
|
features.append(pcs)
|
|
@@ -203,6 +236,9 @@ class SimpleMatrixFeatures:
|
|
|
203
236
|
features = np.concatenate(features, axis=1)
|
|
204
237
|
else:
|
|
205
238
|
features = np.empty((n, 0))
|
|
239
|
+
keep = np.atleast_1d(np.isfinite(features).all(axis=1))
|
|
240
|
+
if keep.all():
|
|
241
|
+
keep = None
|
|
206
242
|
return cls(
|
|
207
243
|
n=n,
|
|
208
244
|
features=features,
|
|
@@ -212,6 +248,7 @@ class SimpleMatrixFeatures:
|
|
|
212
248
|
xyza=xyza,
|
|
213
249
|
amplitudes=amp,
|
|
214
250
|
signed_amplitudes=samp,
|
|
251
|
+
keep_mask=keep,
|
|
215
252
|
)
|
|
216
253
|
|
|
217
254
|
|
|
@@ -331,7 +368,18 @@ def _allfinite(x):
|
|
|
331
368
|
return np.isfinite(x).all()
|
|
332
369
|
|
|
333
370
|
|
|
334
|
-
def
|
|
371
|
+
def _check_numbers(name: str, x: np.ndarray, raise_for_numerics=False):
|
|
335
372
|
if isinstance(x, torch.Tensor):
|
|
336
373
|
x = x.numpy(force=True)
|
|
337
|
-
|
|
374
|
+
|
|
375
|
+
npinf = np.isposinf(x).sum()
|
|
376
|
+
nninf = np.isneginf(x).sum()
|
|
377
|
+
nna = np.isnan(x).sum()
|
|
378
|
+
if not (npinf + nninf + nna):
|
|
379
|
+
return
|
|
380
|
+
|
|
381
|
+
err = f"{name}: {npinf} +inf, {nninf} -inf, {nna} nan."
|
|
382
|
+
if raise_for_numerics:
|
|
383
|
+
raise ValueError(err)
|
|
384
|
+
else:
|
|
385
|
+
logger.warn(err)
|
|
@@ -2683,7 +2683,9 @@ class TruncatedMixtureModel(BaseMixtureModel):
|
|
|
2683
2683
|
# support of the likelihoods to drift away slightly from the weights',
|
|
2684
2684
|
# and this can lead to slightly negative elbo changes. giving a little
|
|
2685
2685
|
# more leeway than in em() here for when to panic about it.
|
|
2686
|
-
|
|
2686
|
+
# in fact, I'm going to comment this assertion out, but leaving the
|
|
2687
|
+
# comment here.
|
|
2688
|
+
# assert (len(elbos) < 2) or (elbos[-1] > elbos[0] - 5e-2)
|
|
2687
2689
|
|
|
2688
2690
|
def score(
|
|
2689
2691
|
self,
|
|
@@ -4381,7 +4383,7 @@ def get_full_neighborhood_data(
|
|
|
4381
4383
|
assert xfull.shape[2] == full_neighborhoods.b.neighborhoods.shape[1]
|
|
4382
4384
|
frank = xfull.shape[1]
|
|
4383
4385
|
if clustering_features_cfg is not None:
|
|
4384
|
-
assert frank
|
|
4386
|
+
assert 0 < frank <= clustering_features_cfg.feature_rank
|
|
4385
4387
|
assert xfull.shape[0] == full_neighborhoods.b.neighborhood_ids.shape[0]
|
|
4386
4388
|
xfull = xfull.view(len(xfull), -1)
|
|
4387
4389
|
xfull = xfull.nan_to_num_()
|
|
@@ -53,10 +53,10 @@ class DARTsortUserConfig:
|
|
|
53
53
|
that there is enough data for clustering. See also subsampling_presence.
|
|
54
54
|
Set to None to disable subsampling."""
|
|
55
55
|
|
|
56
|
-
subsampling_presence: Annotated[float, Field(gt=0.0, le=1.0)] = 0.
|
|
56
|
+
subsampling_presence: Annotated[float, Field(gt=0.0, le=1.0)] = 0.025
|
|
57
57
|
"""Early detection steps which have already found `subsampling_spikes`
|
|
58
58
|
spikes are only allowed to end early if they additionally cover this
|
|
59
|
-
fraction of the recording, to make sure there's
|
|
59
|
+
fraction of the recording, to make sure there's some coverage of
|
|
60
60
|
conditions for template estimation."""
|
|
61
61
|
|
|
62
62
|
matching_iterations: int = 1
|
|
@@ -379,7 +379,7 @@ class DeveloperConfig(DARTsortUserConfig):
|
|
|
379
379
|
# store extra intermediates@
|
|
380
380
|
save_subtracted_waveforms: bool = False
|
|
381
381
|
save_collisioncleaned_waveforms: bool = False
|
|
382
|
-
|
|
382
|
+
always_save_detailed_features: bool = False
|
|
383
383
|
precomputed_templates_npz: str | None = None
|
|
384
384
|
save_everything_on_error: bool = False
|
|
385
385
|
link_from: str | None = None
|
|
@@ -108,6 +108,8 @@ class DARTsortAnalysis:
|
|
|
108
108
|
"localizations",
|
|
109
109
|
"positions",
|
|
110
110
|
),
|
|
111
|
+
compute_extensions=("correlograms",),
|
|
112
|
+
estimate_si_sparsity=False,
|
|
111
113
|
try_backup_amplitudes=("amplitudes", "ptp_amplitudes"),
|
|
112
114
|
quality_metrics=(
|
|
113
115
|
"sliding_rp_violation",
|
|
@@ -288,6 +290,8 @@ class DARTsortAnalysis:
|
|
|
288
290
|
motion=motion,
|
|
289
291
|
features_cfg=clustering_features_cfg,
|
|
290
292
|
compute_extensions_if_templates=None,
|
|
293
|
+
compute_extensions=compute_extensions,
|
|
294
|
+
estimate_si_sparsity=estimate_si_sparsity,
|
|
291
295
|
)
|
|
292
296
|
analyzer.compute("quality_metrics", metric_names=quality_metrics)
|
|
293
297
|
qc_df = analyzer.get_extension("quality_metrics").get_data()
|
|
@@ -379,7 +383,10 @@ class DARTsortAnalysis:
|
|
|
379
383
|
return self._summary_df
|
|
380
384
|
df = self.quality_df.copy()
|
|
381
385
|
uids = self.sorting.unit_ids
|
|
382
|
-
|
|
386
|
+
if self.template_data is not None:
|
|
387
|
+
df["ds_unit_amplitudes"] = pd.Series(
|
|
388
|
+
index=uids, data=self.unit_amplitudes()
|
|
389
|
+
)
|
|
383
390
|
df["ds_firing_rates"] = pd.Series(index=uids, data=self.firing_rates())
|
|
384
391
|
self._summary_df = df
|
|
385
392
|
return df
|
|
@@ -68,6 +68,7 @@ class DARTsortGroundTruthComparison:
|
|
|
68
68
|
self._unsorted_detection = self._greedy_confusion = None
|
|
69
69
|
if self.compute_unsorted_recall:
|
|
70
70
|
self._calculate_greedy_confusion_and_detection()
|
|
71
|
+
self._matched_and_missed_cache = {}
|
|
71
72
|
self._unit_info_cache = {}
|
|
72
73
|
|
|
73
74
|
def _check(self):
|
|
@@ -135,12 +136,12 @@ class DARTsortGroundTruthComparison:
|
|
|
135
136
|
return res
|
|
136
137
|
|
|
137
138
|
def _unit_info_dataframe(self, force_distances=False, perf_only=False):
|
|
138
|
-
amplitudes = self.gt_analysis.unit_amplitudes()
|
|
139
139
|
firing_rates = self.gt_analysis.firing_rates()
|
|
140
140
|
df = self.comparison.get_performance()
|
|
141
141
|
assert isinstance(df, pd.DataFrame)
|
|
142
142
|
df = df.astype(float) # not sure what the problem was...
|
|
143
|
-
|
|
143
|
+
if self.gt_analysis.template_data is not None:
|
|
144
|
+
df["gt_ptp_amplitude"] = self.gt_analysis.unit_amplitudes()
|
|
144
145
|
df["gt_firing_rate"] = firing_rates
|
|
145
146
|
df["gt_sorter_name"] = self.gt_analysis.name or ""
|
|
146
147
|
df["tested_sorter_name"] = self.tested_analysis.name or ""
|
|
@@ -338,6 +339,8 @@ class DARTsortGroundTruthComparison:
|
|
|
338
339
|
assert self._unsorted_detection.shape == (gtns,)
|
|
339
340
|
|
|
340
341
|
def matched_and_missed(self, gt_unit):
|
|
342
|
+
if gt_unit in self._matched_and_missed_cache:
|
|
343
|
+
return self._matched_and_missed_cache[gt_unit]
|
|
341
344
|
(gt_spike_labels,) = self.comparison.get_labels1(gt_unit)
|
|
342
345
|
assert self.gt_analysis.sorting.labels is not None
|
|
343
346
|
in_gt_unit = self.gt_si_inds[
|
|
@@ -346,7 +349,9 @@ class DARTsortGroundTruthComparison:
|
|
|
346
349
|
matched_gt_mask = gt_spike_labels == "TP"
|
|
347
350
|
matched_gt_indices = in_gt_unit[matched_gt_mask]
|
|
348
351
|
only_gt_indices = in_gt_unit[np.logical_not(matched_gt_mask)]
|
|
349
|
-
|
|
352
|
+
ret = in_gt_unit, matched_gt_indices, only_gt_indices
|
|
353
|
+
self._matched_and_missed_cache[gt_unit] = ret
|
|
354
|
+
return ret
|
|
350
355
|
|
|
351
356
|
def get_spikes_by_category(self, gt_unit, tested_unit=None):
|
|
352
357
|
if tested_unit is None:
|
|
@@ -88,8 +88,10 @@ class GrabAndFeaturize(BasePeeler):
|
|
|
88
88
|
skip_features: bool = False,
|
|
89
89
|
n_resid_snips: int | None = None,
|
|
90
90
|
to_cpu: bool = True,
|
|
91
|
+
**peel_kw,
|
|
91
92
|
) -> "PeelingBatchResult":
|
|
92
93
|
"""Override process_chunk to skip empties."""
|
|
94
|
+
assert not peel_kw
|
|
93
95
|
if chunk_end_samples is None:
|
|
94
96
|
chunk_end_samples = min(
|
|
95
97
|
self.recording.get_num_samples(),
|
|
@@ -24,7 +24,7 @@ from ..util.internal_config import (
|
|
|
24
24
|
)
|
|
25
25
|
from ..util.logging_util import get_logger
|
|
26
26
|
from ..util.motion import MotionInfo
|
|
27
|
-
from ..util.waveform_util import make_channel_index
|
|
27
|
+
from ..util.waveform_util import full_channel_index, make_channel_index
|
|
28
28
|
from .matching_util import (
|
|
29
29
|
ChunkTemplateData,
|
|
30
30
|
MatchingPeaks,
|
|
@@ -260,6 +260,7 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
|
|
|
260
260
|
return_residual=False,
|
|
261
261
|
return_waveforms=True,
|
|
262
262
|
return_conv=False,
|
|
263
|
+
return_clean_waveforms=False,
|
|
263
264
|
) -> PeelingBatchResult:
|
|
264
265
|
assert self.matching_templates is not None
|
|
265
266
|
# get chunk center time and template info at that time
|
|
@@ -288,6 +289,7 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
|
|
|
288
289
|
return_residual=return_residual,
|
|
289
290
|
return_conv=return_conv,
|
|
290
291
|
return_collisioncleaned_waveforms=return_waveforms,
|
|
292
|
+
return_clean_waveforms=return_clean_waveforms,
|
|
291
293
|
)
|
|
292
294
|
|
|
293
295
|
# process spike times and create return result
|
|
@@ -304,9 +306,11 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
|
|
|
304
306
|
self,
|
|
305
307
|
traces: Tensor,
|
|
306
308
|
chunk_template_data: ChunkTemplateData,
|
|
309
|
+
*,
|
|
307
310
|
left_margin=0,
|
|
308
311
|
right_margin=0,
|
|
309
312
|
return_collisioncleaned_waveforms=True,
|
|
313
|
+
return_clean_waveforms=False,
|
|
310
314
|
return_residual=False,
|
|
311
315
|
return_conv=False,
|
|
312
316
|
unit_mask=None,
|
|
@@ -475,6 +479,14 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
|
|
|
475
479
|
assert self.p.channel_selection == "template"
|
|
476
480
|
channels = chunk_template_data.main_channels[peaks.template_inds]
|
|
477
481
|
waveforms = None
|
|
482
|
+
if return_clean_waveforms:
|
|
483
|
+
ci_full = full_channel_index(self.b.channel_index.shape[0], True)
|
|
484
|
+
ci_full = ci_full.to(self.b.channel_index)
|
|
485
|
+
res["clean_waveforms"] = chunk_template_data.get_clean_waveforms(
|
|
486
|
+
peaks=peaks,
|
|
487
|
+
channels=channels,
|
|
488
|
+
channel_index=ci_full,
|
|
489
|
+
)
|
|
478
490
|
res["channels"] = channels
|
|
479
491
|
if return_collisioncleaned_waveforms:
|
|
480
492
|
assert waveforms is not None
|
|
@@ -9,6 +9,7 @@ from pathlib import Path
|
|
|
9
9
|
from sys import getrefcount
|
|
10
10
|
from threading import Lock, local
|
|
11
11
|
from typing import Any, Sequence, TypedDict
|
|
12
|
+
from warnings import catch_warnings, filterwarnings
|
|
12
13
|
|
|
13
14
|
import h5py
|
|
14
15
|
import numpy as np
|
|
@@ -496,6 +497,7 @@ class BasePeeler(BModule):
|
|
|
496
497
|
skip_features: bool = False,
|
|
497
498
|
n_resid_snips: int | None = None,
|
|
498
499
|
to_cpu: bool = True,
|
|
500
|
+
**peel_kw,
|
|
499
501
|
) -> "PeelingBatchResult":
|
|
500
502
|
"""Grab, peel, and featurize a chunk, returning a dict of numpy arrays
|
|
501
503
|
|
|
@@ -512,6 +514,7 @@ class BasePeeler(BModule):
|
|
|
512
514
|
right_margin=right_margin,
|
|
513
515
|
return_waveforms=return_waveforms,
|
|
514
516
|
return_residual=return_residual or bool(n_resid_snips),
|
|
517
|
+
**peel_kw,
|
|
515
518
|
)
|
|
516
519
|
chunk_result = self.featurize_chunk_result(
|
|
517
520
|
peel_result=peel_result,
|
|
@@ -537,8 +540,17 @@ class BasePeeler(BModule):
|
|
|
537
540
|
channel_indices=None,
|
|
538
541
|
margin=self.chunk_margin_samples,
|
|
539
542
|
)
|
|
543
|
+
assert isinstance(chunk, np.ndarray)
|
|
540
544
|
device = self.b.channel_index.device
|
|
541
|
-
|
|
545
|
+
if device.type == "cpu":
|
|
546
|
+
chunk = torch.tensor(chunk, device=device, dtype=self.dtype)
|
|
547
|
+
else:
|
|
548
|
+
# we have to copy anyway to go to device, so working around the
|
|
549
|
+
# torch warning here. this way, can avoid blocking.
|
|
550
|
+
with catch_warnings():
|
|
551
|
+
filterwarnings("ignore", message="The given NumPy array is not ")
|
|
552
|
+
chunk = torch.from_numpy(chunk)
|
|
553
|
+
chunk = chunk.to(device=device, dtype=self.dtype, non_blocking=True)
|
|
542
554
|
return chunk, chunk_end_samples, left_margin, right_margin
|
|
543
555
|
|
|
544
556
|
def featurize_chunk_result(
|
|
@@ -20,6 +20,7 @@ from ..util.internal_config import (
|
|
|
20
20
|
)
|
|
21
21
|
from ..util.job_util import ensure_computation_config
|
|
22
22
|
from ..util.spiketorch import grab_spikes, ptp, subtract_spikes_
|
|
23
|
+
from ..util.torch_util import torch_compile
|
|
23
24
|
from ..util.waveform_util import get_relative_subset, make_channel_index
|
|
24
25
|
from .peel_base import PeelingBatchResult
|
|
25
26
|
|
|
@@ -89,46 +90,32 @@ def check_residual_decrease(
|
|
|
89
90
|
return None, {}
|
|
90
91
|
assert orig_wfs is not None
|
|
91
92
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
# spatial mul -- putting temporal dim last here
|
|
104
|
-
orig_wfs = W.bmm(orig_wfs.mT)
|
|
105
|
-
dn_wfs = W.bmm(dn_wfs.mT)
|
|
106
|
-
|
|
107
|
-
# temporal conv if needed
|
|
108
|
-
if whitening_kernel is not None:
|
|
109
|
-
*shp, t = orig_wfs.shape
|
|
110
|
-
k = whitening_kernel[None, None]
|
|
111
|
-
orig_wfs = F.conv1d(orig_wfs.view(-1, 1, t), k, padding="same")
|
|
112
|
-
dn_wfs = F.conv1d(dn_wfs.view(-1, 1, t), k, padding="same")
|
|
113
|
-
orig_wfs = orig_wfs.view(*shp, t)
|
|
114
|
-
dn_wfs = dn_wfs.view(*shp, t)
|
|
93
|
+
orig_wfs = flatten_denan_and_whiten_batched(
|
|
94
|
+
orig_wfs,
|
|
95
|
+
channels,
|
|
96
|
+
local_whiteners,
|
|
97
|
+
whitening_kernel,
|
|
98
|
+
overwrite=overwrite_orig_waveforms,
|
|
99
|
+
)
|
|
100
|
+
dn_wfs = flatten_denan_and_whiten_batched(
|
|
101
|
+
dn_wfs, channels, local_whiteners, whitening_kernel
|
|
102
|
+
)
|
|
115
103
|
|
|
116
104
|
if decrease_objective == "deconv":
|
|
117
105
|
if overwrite_orig_waveforms:
|
|
118
|
-
buf = orig_wfs.mul_(dn_wfs)
|
|
119
|
-
conv = buf.sum(dim=
|
|
106
|
+
buf = orig_wfs.mul_(dn_wfs)
|
|
107
|
+
conv = buf.sum(dim=1)
|
|
120
108
|
torch.square(dn_wfs, out=buf)
|
|
121
|
-
norm = buf.
|
|
109
|
+
norm = buf.sum(dim=1)
|
|
122
110
|
else:
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
norm = dn_wfs.square_().sum(dim=(1, 2))
|
|
111
|
+
conv = (orig_wfs * dn_wfs).sum(dim=1)
|
|
112
|
+
norm = dn_wfs.square_().sum(dim=1)
|
|
126
113
|
reduction = conv.mul_(2.0).sub_(norm)
|
|
127
114
|
threshold = threshold**2
|
|
128
115
|
elif decrease_objective in ("norm", "normsq"):
|
|
129
|
-
orig_decobj = orig_wfs.square().sum(dim=
|
|
130
|
-
orig_wfs = orig_wfs.sub_(dn_wfs)
|
|
131
|
-
new_decobj = orig_wfs.square_().sum(dim=
|
|
116
|
+
orig_decobj = orig_wfs.square().sum(dim=1)
|
|
117
|
+
orig_wfs = orig_wfs.sub_(dn_wfs)
|
|
118
|
+
new_decobj = orig_wfs.square_().sum(dim=1)
|
|
132
119
|
if decrease_objective == "norm":
|
|
133
120
|
orig_decobj = orig_decobj.sqrt_()
|
|
134
121
|
new_decobj = new_decobj.sqrt_()
|
|
@@ -147,6 +134,51 @@ def check_residual_decrease(
|
|
|
147
134
|
return keep, features
|
|
148
135
|
|
|
149
136
|
|
|
137
|
+
@torch_compile
|
|
138
|
+
def flatten_denan_and_whiten_batched(
|
|
139
|
+
wfs: torch.Tensor,
|
|
140
|
+
channels: torch.Tensor | None,
|
|
141
|
+
local_whiteners: Tensor | None,
|
|
142
|
+
whitening_kernel: Tensor | None,
|
|
143
|
+
overwrite: bool = False,
|
|
144
|
+
batch_size: int = 4096,
|
|
145
|
+
):
|
|
146
|
+
N = wfs.shape[0]
|
|
147
|
+
T = wfs.shape[1]
|
|
148
|
+
C = wfs.shape[2]
|
|
149
|
+
if overwrite:
|
|
150
|
+
wfs = wfs.nan_to_num_()
|
|
151
|
+
else:
|
|
152
|
+
wfs = wfs.nan_to_num()
|
|
153
|
+
|
|
154
|
+
if local_whiteners is None:
|
|
155
|
+
return wfs.view(N, -1)
|
|
156
|
+
|
|
157
|
+
assert channels is not None
|
|
158
|
+
|
|
159
|
+
W = local_whiteners[channels]
|
|
160
|
+
wfs = W.bmm(wfs.mT)
|
|
161
|
+
if whitening_kernel is None:
|
|
162
|
+
return wfs.view(N, -1)
|
|
163
|
+
|
|
164
|
+
k = whitening_kernel[None, None]
|
|
165
|
+
NC = N * C
|
|
166
|
+
wfs = wfs.view(NC, 1, T)
|
|
167
|
+
|
|
168
|
+
for i0 in range(0, NC, batch_size):
|
|
169
|
+
i1 = min(NC, i0 + batch_size)
|
|
170
|
+
nb = i1 - i0
|
|
171
|
+
y = wfs[i0:i1]
|
|
172
|
+
if nb < batch_size:
|
|
173
|
+
y = F.pad(y, (0, 0, 0, 0, 0, batch_size - nb))
|
|
174
|
+
z = F.conv1d(y, k, padding="same")
|
|
175
|
+
if nb < batch_size:
|
|
176
|
+
z = z[:nb]
|
|
177
|
+
wfs[i0:i1] = z
|
|
178
|
+
|
|
179
|
+
return wfs.view(N, -1)
|
|
180
|
+
|
|
181
|
+
|
|
150
182
|
ChunkSubtractionResult = namedtuple(
|
|
151
183
|
"ChunkSubtractionResult",
|
|
152
184
|
[
|