DARTsort 0.5.6__tar.gz → 0.5.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.6 → dartsort-0.5.7}/PKG-INFO +1 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/pyproject.toml +1 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/agglomerate.py +246 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/cluster_util.py +41 -4
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/kmeans.py +2 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/mixture.py +76 -24
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/modes.py +3 -42
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/refine_util.py +1 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/config.py +12 -3
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/analysis.py +39 -15
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/comparison.py +26 -8
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/hybrid_util.py +1 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/simkit.py +3 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/main.py +22 -2
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching.py +2 -2
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/pairwise_util.py +6 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/peel_base.py +1 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/reduction_template.py +1 -6
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/get_templates.py +2 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/templates.py +49 -2
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/_multichan_denoiser_kit.py +15 -2
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/decollider.py +17 -3
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/enforce_decrease.py +2 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/mixture_classifier.py +1 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/reduction.py +57 -45
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/data_util.py +28 -10
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/internal_config.py +32 -6
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/interpolation_util.py +3 -3
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/logging_util.py +4 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/nn_util.py +2 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/spiketorch.py +4 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/gt.py +39 -8
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/mixture.py +2 -2
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/recanim.py +2 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/scatterplots.py +18 -14
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/unit.py +38 -6
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/unit_comparison.py +34 -9
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/versus.py +57 -9
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/vismain.py +79 -8
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/waveforms.py +5 -3
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/SOURCES.txt +1 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/conftest.py +3 -3
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_alignment.py +48 -0
- dartsort-0.5.7/tests/test_cluster_util.py +83 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_clustering.py +5 -5
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_data_util.py +2 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_density.py +27 -10
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_detect.py +3 -3
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_enforce_decrease.py +2 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_modeling.py +13 -5
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_spiketorch.py +2 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_torch_optimization_util.py +2 -1
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_util.py +1 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_waveform_util.py +2 -2
- {dartsort-0.5.6 → dartsort-0.5.7}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/.gitignore +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/README.md +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/docs/index.md +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/docs/main_api.md +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/environment.yml +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/.keep +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/basic_integration_test.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/ci diag.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/dartsort_dredge_demo.ipynb +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/dartsort_dredge_demo.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/data test.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/dead reckoning.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/denoise np1.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/denoise np2.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/exploratory.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/geom debug.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/hdbscan_clustering_5min.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/hybrid 10.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/hybrid cluster.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/hybrid_11.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/initial training.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/make hybrid.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/mgh.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/np1 feats.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/np2 processing version c.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/paperfigs.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/pdists.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/postcheck.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/ptp vae.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/re clust.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/reg fig loop.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/reg_sim_exp.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/relocating.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/relocating_all.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/restats +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/sortStinger1.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/sortStinger3.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/sortStinger4.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/sortStinger5.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/sortStinger6.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/spike farm.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/ss_ensemble_sim.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/ss_ensemble_sim_drift.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/ss_sim.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/stan diagnostics.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/storytelling c.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/storytelling.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/debug_matching.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_agreement.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_amplitude_scaling.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_batch_optim.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_ibme.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_mm_initialization.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_normxcorr1d.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_reassignment.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/tests/test_relocation.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/notebook/thomas.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/setup.cfg +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/clustering.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/clustering_features.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/peel_lib.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/subtract.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/pipeline.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/transform_base.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/main_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/motion.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/noise_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/analysis_plots.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_config.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_linalg.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_matching.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_simkit.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_subtract.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_templates.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_threshold.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/tests/test_transform.py +0 -0
- {dartsort-0.5.6 → dartsort-0.5.7}/zensical.toml +0 -0
|
@@ -177,7 +177,13 @@ def agglomerate(
|
|
|
177
177
|
threshold=0.5,
|
|
178
178
|
link=template_merge_cfg.linkage,
|
|
179
179
|
)
|
|
180
|
-
|
|
180
|
+
|
|
181
|
+
agg_sorting = combine_gmm_scores(agg_sorting, new_ids=new_ids)
|
|
182
|
+
|
|
183
|
+
agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
|
|
184
|
+
|
|
185
|
+
agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
|
|
186
|
+
new_ids = reorder[new_ids]
|
|
181
187
|
|
|
182
188
|
return Agglomeration(
|
|
183
189
|
agglomerated_sorting=agg_sorting,
|
|
@@ -625,3 +631,242 @@ def firing_corr(sorting: DARTsortSorting, dt: float, method="binsqrt"):
|
|
|
625
631
|
fr = np.sqrt(fr.values)
|
|
626
632
|
|
|
627
633
|
return np.corrcoef(fr, rowvar=False)
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def combine_gmm_scores(
|
|
637
|
+
sorting: DARTsortSorting, new_ids: np.ndarray, old_prefix="gmm", new_prefix="merged"
|
|
638
|
+
) -> DARTsortSorting:
|
|
639
|
+
"""If new_ids merges units, return a sorting with merged likelihoods."""
|
|
640
|
+
candidates = getattr(sorting, f"{old_prefix}_candidates", None)
|
|
641
|
+
responsibilities = getattr(sorting, f"{old_prefix}_responsibilities", None)
|
|
642
|
+
logliks = getattr(sorting, f"{old_prefix}_log_liks", None)
|
|
643
|
+
|
|
644
|
+
havec = candidates is not None
|
|
645
|
+
haver = responsibilities is not None
|
|
646
|
+
havel = logliks is not None
|
|
647
|
+
assert all([havec, haver, havel]) or not any([havec, havel, haver])
|
|
648
|
+
if not havec:
|
|
649
|
+
return sorting
|
|
650
|
+
assert candidates is not None
|
|
651
|
+
assert responsibilities is not None
|
|
652
|
+
assert logliks is not None
|
|
653
|
+
|
|
654
|
+
# check that new_ids is a merge
|
|
655
|
+
assert (new_ids >= 0).all()
|
|
656
|
+
unique_new_ids, new_id_counts = np.unique(new_ids, return_counts=True)
|
|
657
|
+
assert unique_new_ids.shape[0] == unique_new_ids.max() + 1 <= new_ids.shape[0]
|
|
658
|
+
if unique_new_ids.shape == new_ids.shape:
|
|
659
|
+
return sorting.ephemeral_replace(
|
|
660
|
+
**{
|
|
661
|
+
f"{new_prefix}_candidates": candidates,
|
|
662
|
+
f"{new_prefix}_responsibilities": responsibilities,
|
|
663
|
+
f"{new_prefix}_logliks": logliks,
|
|
664
|
+
}
|
|
665
|
+
)
|
|
666
|
+
|
|
667
|
+
# check invariants at the top
|
|
668
|
+
if responsibilities.shape[1] > 2:
|
|
669
|
+
assert np.all(np.diff(responsibilities[:, :-1], axis=1) <= 0)
|
|
670
|
+
assert np.greater_equal(np.isneginf(logliks[:, :-1]), candidates == -1).all()
|
|
671
|
+
if sorting.labels is not None:
|
|
672
|
+
assert np.all(
|
|
673
|
+
np.logical_or(
|
|
674
|
+
sorting.labels < 0, sorting.labels == new_ids[candidates[:, 0]]
|
|
675
|
+
)
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
# two steps: first merge, then sort
|
|
679
|
+
# merge candidates
|
|
680
|
+
new_ids_ = np.pad(new_ids, [(0, 1)], constant_values=-1)
|
|
681
|
+
orig_bye = candidates < 0
|
|
682
|
+
nbye = orig_bye.sum()
|
|
683
|
+
cand = np.where(orig_bye, new_ids.shape[0], candidates)
|
|
684
|
+
cand = new_ids_[cand]
|
|
685
|
+
assert (cand < 0).sum() >= nbye
|
|
686
|
+
|
|
687
|
+
# deduplicate
|
|
688
|
+
mergedr = responsibilities[:, : cand.shape[1]].copy()
|
|
689
|
+
mergedl = logliks[:, : cand.shape[1]].copy()
|
|
690
|
+
_combine_loop(cand, new_id_counts, mergedr, mergedl)
|
|
691
|
+
|
|
692
|
+
# now re-sort
|
|
693
|
+
order = np.argsort(-mergedl, axis=1, kind="stable")
|
|
694
|
+
cand = np.take_along_axis(cand, axis=1, indices=order)
|
|
695
|
+
mergedr = np.take_along_axis(mergedr, axis=1, indices=order)
|
|
696
|
+
mergedl = np.take_along_axis(mergedl, axis=1, indices=order)
|
|
697
|
+
mergedl = np.concatenate([mergedl, logliks[:, cand.shape[1] :]], axis=1)
|
|
698
|
+
mergedr = np.concatenate([mergedr, responsibilities[:, cand.shape[1] :]], axis=1)
|
|
699
|
+
|
|
700
|
+
# check invariants at the bottom
|
|
701
|
+
if mergedr.shape[1] > 2:
|
|
702
|
+
assert np.all(np.diff(mergedr[:, : cand.shape[1]], axis=1) <= 0)
|
|
703
|
+
assert np.greater_equal(np.isneginf(mergedl[:, : cand.shape[1]]), cand == -1).all()
|
|
704
|
+
assert (cand < 0).sum() >= nbye
|
|
705
|
+
if sorting.labels is not None:
|
|
706
|
+
changed = sorting.labels != cand[:, 0]
|
|
707
|
+
changed = changed[sorting.labels >= 0]
|
|
708
|
+
logger.dartsortdebug(
|
|
709
|
+
f"Mixture component aggregation changes {100 * changed.mean():0.2f}"
|
|
710
|
+
f"% of spike labels ({changed.sum().item()} spikes)."
|
|
711
|
+
)
|
|
712
|
+
return sorting.ephemeral_replace(
|
|
713
|
+
labels=np.where(mergedl[:, 0] >= mergedl[:, -1], cand[:, 0], -1),
|
|
714
|
+
**{
|
|
715
|
+
f"{new_prefix}_candidates": cand,
|
|
716
|
+
f"{new_prefix}_responsibilities": mergedr,
|
|
717
|
+
f"{new_prefix}_logliks": mergedl,
|
|
718
|
+
},
|
|
719
|
+
)
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
@numba.njit(parallel=True)
|
|
723
|
+
def _combine_loop(
|
|
724
|
+
cand: np.ndarray,
|
|
725
|
+
new_id_counts: np.ndarray,
|
|
726
|
+
mergedr: np.ndarray,
|
|
727
|
+
mergedl: np.ndarray,
|
|
728
|
+
):
|
|
729
|
+
for s in numba.prange(cand.shape[0]): # ty: ignore
|
|
730
|
+
rcand = cand[s]
|
|
731
|
+
for j in range(cand.shape[1] - 1):
|
|
732
|
+
ncandj = rcand[j]
|
|
733
|
+
if ncandj < 0 or new_id_counts[ncandj] <= 1:
|
|
734
|
+
continue
|
|
735
|
+
|
|
736
|
+
eq_ncandj = rcand[j + 1 :] == ncandj
|
|
737
|
+
if eq_ncandj.sum() <= 1:
|
|
738
|
+
continue
|
|
739
|
+
|
|
740
|
+
rsum = mergedr[s, j]
|
|
741
|
+
lsum = mergedl[s, j]
|
|
742
|
+
for i, k in enumerate(range(j + 1, cand.shape[1])):
|
|
743
|
+
if not eq_ncandj[i]:
|
|
744
|
+
continue
|
|
745
|
+
cand[s, k] = -1
|
|
746
|
+
|
|
747
|
+
if mergedl[s, k] == -np.inf:
|
|
748
|
+
continue
|
|
749
|
+
rsum += mergedr[s, k]
|
|
750
|
+
lsum = np.logaddexp(lsum, mergedl[s, k])
|
|
751
|
+
|
|
752
|
+
mergedr[s, k] = 0.0
|
|
753
|
+
mergedl[s, k] = -np.inf
|
|
754
|
+
|
|
755
|
+
mergedr[s, j] = rsum
|
|
756
|
+
mergedl[s, j] = lsum
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def deduplicate_spikes(
|
|
760
|
+
sorting: DARTsortSorting,
|
|
761
|
+
radius_ms: float = -1.0,
|
|
762
|
+
score_by=("merged_logliks", "gmm_logliks", "scores"),
|
|
763
|
+
) -> DARTsortSorting:
|
|
764
|
+
"""The lower-scoring of any spikes within radius_samples of each other is relabeled to -1.
|
|
765
|
+
|
|
766
|
+
scores are grabbed from the first entry of score_by which is a valid property of the sorting.
|
|
767
|
+
If it's mu
|
|
768
|
+
"""
|
|
769
|
+
if radius_ms < 0 or sorting.labels is None:
|
|
770
|
+
return sorting
|
|
771
|
+
|
|
772
|
+
radius_samples = WaveformConfig.ms_to_samples(
|
|
773
|
+
ms=radius_ms,
|
|
774
|
+
sampling_frequency=sorting.sampling_frequency,
|
|
775
|
+
)
|
|
776
|
+
assert radius_samples >= 0
|
|
777
|
+
|
|
778
|
+
new_labels = sorting.labels.copy()
|
|
779
|
+
scores = None
|
|
780
|
+
for sck in score_by:
|
|
781
|
+
scores = getattr(sorting, sck, None)
|
|
782
|
+
if scores is not None:
|
|
783
|
+
logger.dartsortdebug(f"deduplicate by score {sck}")
|
|
784
|
+
break
|
|
785
|
+
if scores is None:
|
|
786
|
+
raise ValueError(f"sorting had none of {score_by}.")
|
|
787
|
+
if scores.ndim == 2:
|
|
788
|
+
scores = scores[:, 0]
|
|
789
|
+
assert scores.ndim == 1
|
|
790
|
+
assert scores.shape == new_labels.shape
|
|
791
|
+
|
|
792
|
+
# handle unsorted times
|
|
793
|
+
tsort = np.argsort(sorting.times_samples)
|
|
794
|
+
new_labels = new_labels[tsort]
|
|
795
|
+
times_samples = sorting.times_samples[tsort]
|
|
796
|
+
scores = scores[tsort]
|
|
797
|
+
|
|
798
|
+
unit_ids = np.unique(new_labels)
|
|
799
|
+
unit_ids = unit_ids[unit_ids >= 0]
|
|
800
|
+
ndrop = 0
|
|
801
|
+
for unit_id in unit_ids:
|
|
802
|
+
in_unit = np.flatnonzero(new_labels == unit_id)
|
|
803
|
+
t = times_samples[in_unit]
|
|
804
|
+
dt = np.diff(t)
|
|
805
|
+
if dt.min() > radius_samples:
|
|
806
|
+
continue
|
|
807
|
+
discard = _dedup_unit(t, dt, scores[in_unit], radius_samples)
|
|
808
|
+
ndrop += discard.sum()
|
|
809
|
+
new_labels[in_unit[discard]] = -1
|
|
810
|
+
|
|
811
|
+
logger.dartsortdebug(f"drop {ndrop}/{len(sorting)} isi violator spikes")
|
|
812
|
+
new_labels = new_labels[np.argsort(tsort)]
|
|
813
|
+
|
|
814
|
+
return sorting.ephemeral_replace(labels=new_labels)
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
def _dedup_unit(
|
|
818
|
+
t: np.ndarray, dt: np.ndarray, scores: np.ndarray, radius: int
|
|
819
|
+
) -> np.ndarray:
|
|
820
|
+
"""Deduplicate a single unit's spike train."""
|
|
821
|
+
discard = np.zeros(t.shape, dtype=bool)
|
|
822
|
+
_dedup_unit_loop(dt, scores, radius, discard)
|
|
823
|
+
return discard
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
@numba.njit
|
|
827
|
+
def _dedup_unit_loop(
|
|
828
|
+
dt: np.ndarray, scores: np.ndarray, radius: int, discard: np.ndarray
|
|
829
|
+
):
|
|
830
|
+
n = scores.shape[0]
|
|
831
|
+
i0 = 0
|
|
832
|
+
while i0 < n - 1:
|
|
833
|
+
if dt[i0] > radius:
|
|
834
|
+
i0 += 1
|
|
835
|
+
continue
|
|
836
|
+
|
|
837
|
+
i1 = i0 + 1
|
|
838
|
+
while i1 < n - 1 and dt[i1] <= radius:
|
|
839
|
+
i1 += 1
|
|
840
|
+
|
|
841
|
+
# now, i0:i1 + 1 is a slice of violators
|
|
842
|
+
# discard until all valid
|
|
843
|
+
score_slice = scores[i0 : i1 + 1]
|
|
844
|
+
dt_slice = dt[i0:i1]
|
|
845
|
+
order = np.argsort(score_slice)
|
|
846
|
+
for oo in order:
|
|
847
|
+
# discard spike i0 + oo
|
|
848
|
+
ii = i0 + oo
|
|
849
|
+
discard[ii] = True
|
|
850
|
+
|
|
851
|
+
# figure out isi after bridging the gap, handle edges
|
|
852
|
+
bridge_isi = 0.0
|
|
853
|
+
if ii < n - 1:
|
|
854
|
+
bridge_isi += dt[ii]
|
|
855
|
+
else:
|
|
856
|
+
bridge_isi = np.inf
|
|
857
|
+
if ii > 0:
|
|
858
|
+
bridge_isi += dt[ii - 1]
|
|
859
|
+
else:
|
|
860
|
+
bridge_isi = np.inf
|
|
861
|
+
|
|
862
|
+
# update dt with bridge isi
|
|
863
|
+
if ii < n - 1:
|
|
864
|
+
dt[ii] = bridge_isi
|
|
865
|
+
if ii > 0:
|
|
866
|
+
dt[ii - 1] = bridge_isi
|
|
867
|
+
|
|
868
|
+
# check if done
|
|
869
|
+
if dt_slice.min() > radius:
|
|
870
|
+
break
|
|
871
|
+
|
|
872
|
+
i0 = i1
|
|
@@ -321,8 +321,29 @@ def combine_disjoint(inds_a, labels_a, inds_b, labels_b):
|
|
|
321
321
|
|
|
322
322
|
|
|
323
323
|
def reorder_by_depth(
|
|
324
|
-
sorting
|
|
325
|
-
|
|
324
|
+
sorting: DARTsortSorting,
|
|
325
|
+
motion: MotionInfo | None = None,
|
|
326
|
+
spatial_footprints: np.ndarray | None = None,
|
|
327
|
+
geom: np.ndarray | None = None,
|
|
328
|
+
centroids: np.ndarray | None = None,
|
|
329
|
+
) -> tuple[DARTsortSorting, np.ndarray]:
|
|
330
|
+
"""Reorder cluster labels so that centroid depth is increasing
|
|
331
|
+
|
|
332
|
+
Parameters
|
|
333
|
+
----------
|
|
334
|
+
sorting : DARTsortSorting
|
|
335
|
+
motion : MotionInfo | None, optional
|
|
336
|
+
spatial_footprints : np.ndarray | None, optional
|
|
337
|
+
geom : np.ndarray | None, optional
|
|
338
|
+
centroids : np.ndarray | None, optional
|
|
339
|
+
|
|
340
|
+
Returns
|
|
341
|
+
-------
|
|
342
|
+
reordered_sorting: DARTsortSorting
|
|
343
|
+
reorder: np.ndarray
|
|
344
|
+
reorder[j] is the new label of original unit j.
|
|
345
|
+
"""
|
|
346
|
+
assert sorting.labels is not None
|
|
326
347
|
kept = np.flatnonzero(sorting.labels >= 0)
|
|
327
348
|
kept_labels = sorting.labels[kept]
|
|
328
349
|
|
|
@@ -355,8 +376,9 @@ def reorder_by_depth(
|
|
|
355
376
|
# this one is some food for thought, lol.
|
|
356
377
|
reorder = np.argsort(np.argsort(centroids, kind="stable"), kind="stable")
|
|
357
378
|
labels[kept] = reorder[kept_labels]
|
|
379
|
+
reordered_sorting = sorting.ephemeral_replace(labels=labels)
|
|
358
380
|
|
|
359
|
-
return
|
|
381
|
+
return reordered_sorting, reorder
|
|
360
382
|
|
|
361
383
|
|
|
362
384
|
def closest_registered_channels(
|
|
@@ -502,7 +524,22 @@ def get_main_channel_pcs(
|
|
|
502
524
|
return features
|
|
503
525
|
|
|
504
526
|
|
|
505
|
-
def decrumb(labels, min_size=5, in_place=False, flatten=True):
|
|
527
|
+
def decrumb(labels: np.ndarray, min_size: int=5, in_place=False, flatten=True):
|
|
528
|
+
"""Remove small units
|
|
529
|
+
|
|
530
|
+
Parameters
|
|
531
|
+
----------
|
|
532
|
+
labels : np.ndarray
|
|
533
|
+
min_size : int
|
|
534
|
+
in_place : bool
|
|
535
|
+
flatten : bool
|
|
536
|
+
Flatten the output label space to be contiguous.
|
|
537
|
+
|
|
538
|
+
Returns
|
|
539
|
+
-------
|
|
540
|
+
labels
|
|
541
|
+
The (flattened) decrumbed labels.
|
|
542
|
+
"""
|
|
506
543
|
kept = np.flatnonzero(labels >= 0)
|
|
507
544
|
labels_kept = labels[kept]
|
|
508
545
|
labels = labels if in_place else labels.copy()
|
|
@@ -99,6 +99,7 @@ def kmeanspp(
|
|
|
99
99
|
)
|
|
100
100
|
j = n_components
|
|
101
101
|
else:
|
|
102
|
+
j = 0
|
|
102
103
|
for j in range(1, n_components):
|
|
103
104
|
if weights is not None:
|
|
104
105
|
torch.mul(dists, weights, out=p)
|
|
@@ -176,6 +177,7 @@ def truncated_kmeans(
|
|
|
176
177
|
it = progrange(n_initializations, desc="kmeans++")
|
|
177
178
|
else:
|
|
178
179
|
it = range(n_initializations)
|
|
180
|
+
_d = None
|
|
179
181
|
for _ in it:
|
|
180
182
|
_c, _l, _d, _p = kmeanspp(
|
|
181
183
|
X,
|
|
@@ -1236,7 +1236,7 @@ class BatchedSpikeData:
|
|
|
1236
1236
|
if self.candidates is None:
|
|
1237
1237
|
return
|
|
1238
1238
|
assert self.rg is not None
|
|
1239
|
-
same_adj = _fill_blank_labels(
|
|
1239
|
+
same_adj, n_new = _fill_blank_labels(
|
|
1240
1240
|
labels=self.candidates[:, 0],
|
|
1241
1241
|
un_adj=self.un_adj,
|
|
1242
1242
|
explore_adj=self.explore_adj,
|
|
@@ -1246,6 +1246,10 @@ class BatchedSpikeData:
|
|
|
1246
1246
|
allow_uncovered=allow_uncovered,
|
|
1247
1247
|
)
|
|
1248
1248
|
assert same_adj is not None
|
|
1249
|
+
if allow_uncovered:
|
|
1250
|
+
assert n_new == 0
|
|
1251
|
+
else:
|
|
1252
|
+
n_units = n_units + n_new
|
|
1249
1253
|
if not same_adj:
|
|
1250
1254
|
logger.dartsortverbose(
|
|
1251
1255
|
"_fill_blank_labels used explore adjacency in bootstrap_candidates."
|
|
@@ -1334,7 +1338,7 @@ class BatchedSpikeData:
|
|
|
1334
1338
|
self.update_adjacency(n_units=None)
|
|
1335
1339
|
assert self.candidates is not None
|
|
1336
1340
|
assert self.rg is not None
|
|
1337
|
-
res = _fill_blank_labels(
|
|
1341
|
+
res, n_new = _fill_blank_labels(
|
|
1338
1342
|
labels=self.candidates[:, 0],
|
|
1339
1343
|
un_adj=self.un_adj,
|
|
1340
1344
|
explore_adj=self.explore_adj,
|
|
@@ -1347,6 +1351,7 @@ class BatchedSpikeData:
|
|
|
1347
1351
|
max_steps=max_steps if max_steps is not None else self.coverage_steps,
|
|
1348
1352
|
)
|
|
1349
1353
|
assert res is None
|
|
1354
|
+
return n_new
|
|
1350
1355
|
|
|
1351
1356
|
def erase_candidates(self):
|
|
1352
1357
|
if self.candidates is not None:
|
|
@@ -1672,6 +1677,9 @@ class TruncatedSpikeData(BatchedSpikeData):
|
|
|
1672
1677
|
)
|
|
1673
1678
|
assert self.candidates is not None
|
|
1674
1679
|
self.candidates[:, 0] = labels
|
|
1680
|
+
if pnoid:
|
|
1681
|
+
_k = labels.amax() + 1 + int((labels < 0).any())
|
|
1682
|
+
assert labels.unique().shape == (_k,)
|
|
1675
1683
|
return self
|
|
1676
1684
|
|
|
1677
1685
|
def update(
|
|
@@ -1846,6 +1854,7 @@ class TruncatedSpikeData(BatchedSpikeData):
|
|
|
1846
1854
|
assert n_units_new <= n_units_orig
|
|
1847
1855
|
|
|
1848
1856
|
if distances is not None:
|
|
1857
|
+
assert distances.shape[0] == distances.shape[1]
|
|
1849
1858
|
assert distances.shape[0] == n_units_new
|
|
1850
1859
|
assert distances.shape[0] <= n_units_orig
|
|
1851
1860
|
assert remapping.mapping.max() + 1 == distances.shape[0]
|
|
@@ -1870,12 +1879,24 @@ class TruncatedSpikeData(BatchedSpikeData):
|
|
|
1870
1879
|
out=self.candidates[:, : self.n_candidates],
|
|
1871
1880
|
)
|
|
1872
1881
|
self.candidates[:, self.n_candidates :] = -1
|
|
1882
|
+
if pnoid and distances is not None:
|
|
1883
|
+
assert self.candidates.amax() < distances.shape[0], 1
|
|
1873
1884
|
self.update_adjacency(n_units_new)
|
|
1874
|
-
self.ensure_coverage(
|
|
1885
|
+
n_new = self.ensure_coverage(
|
|
1875
1886
|
allow_new_units=not allow_uncovered,
|
|
1876
1887
|
allow_uncovered=allow_uncovered,
|
|
1877
1888
|
max_steps=0 if allow_uncovered else None,
|
|
1878
1889
|
)
|
|
1890
|
+
if allow_uncovered:
|
|
1891
|
+
assert n_new == 0
|
|
1892
|
+
elif distances is not None:
|
|
1893
|
+
d = distances.new_full(
|
|
1894
|
+
(n_units_new + n_new, n_units_new + n_new), torch.inf
|
|
1895
|
+
)
|
|
1896
|
+
d[:n_units_new, :n_units_new] = distances
|
|
1897
|
+
d[n_units_new:, n_units_new:].fill_diagonal_(0.0)
|
|
1898
|
+
if pnoid and distances is not None:
|
|
1899
|
+
assert self.candidates.amax() < distances.shape[0], 2
|
|
1879
1900
|
if distances is not None:
|
|
1880
1901
|
_, lut = self.update(new_top_candidates=None, distances=distances)
|
|
1881
1902
|
else:
|
|
@@ -2515,6 +2536,7 @@ class TruncatedMixtureModel(BaseMixtureModel):
|
|
|
2515
2536
|
assert self.b.noise_log_prop.isfinite().all()
|
|
2516
2537
|
else:
|
|
2517
2538
|
assert stats.noise_N is None
|
|
2539
|
+
lp = None
|
|
2518
2540
|
|
|
2519
2541
|
# rank 0 case: mean only
|
|
2520
2542
|
if self.signal_rank == 0:
|
|
@@ -2537,6 +2559,7 @@ class TruncatedMixtureModel(BaseMixtureModel):
|
|
|
2537
2559
|
if pnoid:
|
|
2538
2560
|
assert soln.isfinite().all()
|
|
2539
2561
|
if not skip_proportions:
|
|
2562
|
+
assert lp is not None
|
|
2540
2563
|
assert lp.isfinite().all()
|
|
2541
2564
|
|
|
2542
2565
|
def fixed_weight_em(
|
|
@@ -4264,21 +4287,49 @@ def get_full_neighborhood_data(
|
|
|
4264
4287
|
|
|
4265
4288
|
rg = np.random.default_rng(rg)
|
|
4266
4289
|
|
|
4267
|
-
#
|
|
4268
|
-
n_fit = refinement_cfg.sampling_cfg.more_waveforms_fit
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4274
|
-
assert fit_indices is not None
|
|
4275
|
-
n_fit = fit_indices.shape[0]
|
|
4276
|
-
|
|
4277
|
-
# data splits: -1 full, 0 train, 2 val
|
|
4290
|
+
# how many waveforms will be used for fitting?
|
|
4291
|
+
n_fit = min(len(sorting), refinement_cfg.sampling_cfg.more_waveforms_fit)
|
|
4292
|
+
val_prop = refinement_cfg.val_proportion
|
|
4293
|
+
train_prop = 1.0 - val_prop
|
|
4294
|
+
fit_prop_of_total = n_fit / len(sorting)
|
|
4295
|
+
|
|
4296
|
+
# data splits: -1 full, 0 train, 1 val
|
|
4278
4297
|
split_mask = np.full(len(sorting), -1, dtype=np.int8)
|
|
4279
|
-
|
|
4280
|
-
|
|
4281
|
-
|
|
4298
|
+
# these are stratified by unit in order to ensure that all units are represented
|
|
4299
|
+
# in at least the train set rather than risking a miss with totally random sampling
|
|
4300
|
+
assert sorting.labels is not None
|
|
4301
|
+
unit_ids, counts = np.unique(sorting.labels, return_counts=True)
|
|
4302
|
+
_pos = unit_ids >= 0
|
|
4303
|
+
counts = counts[_pos]
|
|
4304
|
+
unit_ids = unit_ids[_pos]
|
|
4305
|
+
fit_counts = np.minimum(
|
|
4306
|
+
counts, np.ceil(counts * fit_prop_of_total).astype(np.int64)
|
|
4307
|
+
)
|
|
4308
|
+
assert fit_counts.dtype.kind == "i"
|
|
4309
|
+
train_counts = np.maximum(train_prop * fit_counts, 1).astype(np.int64)
|
|
4310
|
+
for uid, train_count, fit_count in zip(unit_ids, train_counts, fit_counts):
|
|
4311
|
+
in_unit = np.flatnonzero(sorting.labels == uid)
|
|
4312
|
+
if in_unit.size > fit_count:
|
|
4313
|
+
fit_ixs = rg.choice(in_unit, size=fit_count)
|
|
4314
|
+
fit_ixs.sort()
|
|
4315
|
+
elif in_unit.size == fit_count:
|
|
4316
|
+
fit_ixs = in_unit
|
|
4317
|
+
else:
|
|
4318
|
+
assert False
|
|
4319
|
+
if fit_ixs.size > train_count:
|
|
4320
|
+
train_ixs = rg.choice(fit_ixs, size=train_count)
|
|
4321
|
+
train_ixs.sort()
|
|
4322
|
+
elif fit_ixs.size == train_count:
|
|
4323
|
+
train_ixs = fit_ixs
|
|
4324
|
+
else:
|
|
4325
|
+
assert False
|
|
4326
|
+
|
|
4327
|
+
# backfill with val set
|
|
4328
|
+
split_mask[fit_ixs] = 1
|
|
4329
|
+
# cover with train
|
|
4330
|
+
split_mask[train_ixs] = 0
|
|
4331
|
+
|
|
4332
|
+
# sparsify index sets
|
|
4282
4333
|
train_indices = torch.asarray(np.flatnonzero(split_mask == 0))
|
|
4283
4334
|
val_indices = torch.asarray(np.flatnonzero(split_mask == 1))
|
|
4284
4335
|
|
|
@@ -5944,9 +5995,9 @@ def _fill_blank_labels(
|
|
|
5944
5995
|
(blank,) = (labels < 0).nonzero(as_tuple=True)
|
|
5945
5996
|
Nblank = blank.numel()
|
|
5946
5997
|
if not Nblank and ensure_coverage_only:
|
|
5947
|
-
return None
|
|
5998
|
+
return None, 0
|
|
5948
5999
|
elif not Nblank:
|
|
5949
|
-
return True
|
|
6000
|
+
return True, 0
|
|
5950
6001
|
|
|
5951
6002
|
# need full coverage of neighborhoods here. would prefer only to branch out with
|
|
5952
6003
|
# explore as needed, so make those probs tiny.
|
|
@@ -5984,8 +6035,9 @@ def _fill_blank_labels(
|
|
|
5984
6035
|
neighb_component_ids = neighb_component_ids - neighb_component_ids.amin()
|
|
5985
6036
|
n_spikes = 0
|
|
5986
6037
|
if pnoid:
|
|
5987
|
-
Kold = labels.max() + 1 + int(
|
|
5988
|
-
|
|
6038
|
+
Kold = labels.max().item() + 1 + int(Nblank > 0)
|
|
6039
|
+
_lshp = labels.unique().shape
|
|
6040
|
+
assert _lshp == (Kold,), f"{Kold=} {_lshp=}"
|
|
5989
6041
|
for j, nid in enumerate(uncovered_neighbs):
|
|
5990
6042
|
(in_nid,) = (neighborhood_ids == nid).nonzero(as_tuple=True)
|
|
5991
6043
|
n_spikes += in_nid.numel()
|
|
@@ -5998,7 +6050,7 @@ def _fill_blank_labels(
|
|
|
5998
6050
|
logger.dartsortverbose(
|
|
5999
6051
|
f"_fill_blank_labels made {n_new_units} new units for {n_spikes} spikes."
|
|
6000
6052
|
)
|
|
6001
|
-
return None
|
|
6053
|
+
return None, n_new_units
|
|
6002
6054
|
|
|
6003
6055
|
if uncovered:
|
|
6004
6056
|
# raise if still_uncovered, warn or log otherwise
|
|
@@ -6011,7 +6063,7 @@ def _fill_blank_labels(
|
|
|
6011
6063
|
)
|
|
6012
6064
|
|
|
6013
6065
|
if ensure_coverage_only:
|
|
6014
|
-
return None
|
|
6066
|
+
return None, n_new_units
|
|
6015
6067
|
|
|
6016
6068
|
for i0 in range(0, Nblank, batch_size):
|
|
6017
6069
|
i1 = min(Nblank, i0 + batch_size)
|
|
@@ -6021,7 +6073,7 @@ def _fill_blank_labels(
|
|
|
6021
6073
|
draws = torch.multinomial(p, 1, replacement=True, generator=gen)
|
|
6022
6074
|
labels[ii] = draws.view(ii.shape[0])
|
|
6023
6075
|
|
|
6024
|
-
return keep_same_adj
|
|
6076
|
+
return keep_same_adj, n_new_units
|
|
6025
6077
|
|
|
6026
6078
|
|
|
6027
6079
|
def _bootstrap_top(
|
|
@@ -42,46 +42,6 @@ def fit_unimodal_right(x, f, weights=None, cut=0, hard=False):
|
|
|
42
42
|
return out
|
|
43
43
|
|
|
44
44
|
|
|
45
|
-
def fit_truncnorm_right(x, f, weights=None, cut=0, hard=False, n_iter=10):
|
|
46
|
-
"""Like above, but fits truncated normal to xs > cut with MoM."""
|
|
47
|
-
from scipy.stats import norm
|
|
48
|
-
|
|
49
|
-
if weights is None:
|
|
50
|
-
weights = np.ones_like(f)
|
|
51
|
-
|
|
52
|
-
# figure out where cut lands and what f(cut) should be
|
|
53
|
-
cuti_left = np.searchsorted(x, cut)
|
|
54
|
-
if cuti_left >= len(x) - 2:
|
|
55
|
-
# everything is to the left... return uniform
|
|
56
|
-
return np.full_like(f, 1 / len(f))
|
|
57
|
-
cuti_right = np.searchsorted(x, cut, side="right") - 1
|
|
58
|
-
if cuti_right <= 1:
|
|
59
|
-
# everything is to the right... fit normal!
|
|
60
|
-
mean = np.average(x, weights=weights)
|
|
61
|
-
var = np.average((x - mean) ** 2, weights=weights)
|
|
62
|
-
return norm.pdf(x, loc=mean, scale=np.sqrt(var))
|
|
63
|
-
# else, in the middle somewhere...
|
|
64
|
-
assert cuti_left in (cuti_right, cuti_right + 1)
|
|
65
|
-
|
|
66
|
-
xcut = x[cuti_right:]
|
|
67
|
-
mean = np.average(xcut, weights=weights[cuti_right:])
|
|
68
|
-
var = np.average((xcut - mean) ** 2, weights=weights[cuti_right:])
|
|
69
|
-
mu = mean
|
|
70
|
-
sigma = std = np.sqrt(var)
|
|
71
|
-
for i in range(n_iter):
|
|
72
|
-
alpha = (cut - mu) / sigma
|
|
73
|
-
phi_alpha = norm.cdf(alpha)
|
|
74
|
-
Z = 1.0 - phi_alpha
|
|
75
|
-
# we have: mean ~ mu + sigma*phi(alpha)/Z
|
|
76
|
-
# and var ~ sigma^2(1+alpha phi(alpha) /Z - (phi(alpha)/Z)^2)
|
|
77
|
-
# implies mu = mean - sigma*phi(alpha)/Z
|
|
78
|
-
# sigma^2 = var/(1+alpha phi(alpha) /Z - (phi(alpha)/Z)^2)
|
|
79
|
-
sigma = np.sqrt(var / (1 + alpha * phi_alpha / Z - (phi_alpha / Z) ** 2))
|
|
80
|
-
mu = mean - sigma * phi_alpha / Z
|
|
81
|
-
|
|
82
|
-
return norm.pdf(x, loc=mu, scale=sigma)
|
|
83
|
-
|
|
84
|
-
|
|
85
45
|
def fit_bimodal_at(x, f, weights=None, cut=0):
|
|
86
46
|
from isosplit import up_down_isotonic_regression # type: ignore
|
|
87
47
|
|
|
@@ -157,6 +117,8 @@ def smoothed_dipscore_at(
|
|
|
157
117
|
|
|
158
118
|
if score_kind == "ks":
|
|
159
119
|
empirical = np.cumsum(densities * spacings)
|
|
120
|
+
else:
|
|
121
|
+
empirical = None
|
|
160
122
|
|
|
161
123
|
if null in ("isotonic", "gcm"):
|
|
162
124
|
orders = (slice(None),)
|
|
@@ -174,8 +136,6 @@ def smoothed_dipscore_at(
|
|
|
174
136
|
dens = fit_unimodal_right(sign * s, d, cut=sign * cut, hard=hard)
|
|
175
137
|
elif null == "isotonic":
|
|
176
138
|
dens = up_down_isotonic_regression(d)
|
|
177
|
-
elif null == "truncnorm":
|
|
178
|
-
dens = fit_truncnorm_right(sign * s, d, cut=sign * cut, hard=hard)
|
|
179
139
|
else:
|
|
180
140
|
assert False
|
|
181
141
|
|
|
@@ -185,6 +145,7 @@ def smoothed_dipscore_at(
|
|
|
185
145
|
|
|
186
146
|
dens_err = (np.abs(dens - densities) * spacings).sum()
|
|
187
147
|
if score_kind == "ks":
|
|
148
|
+
assert empirical is not None
|
|
188
149
|
my_score = np.abs(empirical - np.cumsum(dens * spacings)).max()
|
|
189
150
|
elif score_kind == "tv":
|
|
190
151
|
my_score = 0.5 * dens_err
|
|
@@ -175,7 +175,7 @@ def pc_merge(
|
|
|
175
175
|
)
|
|
176
176
|
sf = means.square_().sum(dim=1).sqrt_() * counts.sqrt()
|
|
177
177
|
sf = sf.numpy(force=True)
|
|
178
|
-
sorting = reorder_by_depth(
|
|
178
|
+
sorting, new_ids = reorder_by_depth(
|
|
179
179
|
sorting, motion=motion, spatial_footprints=sf, geom=motion.rgeom
|
|
180
180
|
)
|
|
181
181
|
return PCMergeResult(sorting=sorting)
|
|
@@ -123,6 +123,13 @@ class DARTsortUserConfig:
|
|
|
123
123
|
alignment_ms: Annotated[float, Field(gt=0)] = 1.5
|
|
124
124
|
"""Largest time shift allowed when re-aligning events."""
|
|
125
125
|
|
|
126
|
+
deduplication_ms: Annotated[float, Field(gt=0)] = 0.5
|
|
127
|
+
"""As a final postprocessing step, only the higher-scoring of any spikes within
|
|
128
|
+
this time radius of each other are kept.
|
|
129
|
+
|
|
130
|
+
If this is negative, it does nothing. If it's 0, exact duplicates are dropped.
|
|
131
|
+
"""
|
|
132
|
+
|
|
126
133
|
# -- thresholds
|
|
127
134
|
peak_sign: Literal["neg", "both", "pos"] = "both"
|
|
128
135
|
"""Allow only troughs or events of both signs when detecting threshold
|
|
@@ -239,7 +246,9 @@ class DeveloperConfig(DARTsortUserConfig):
|
|
|
239
246
|
n_residual_snips: int = 4 * 4096
|
|
240
247
|
|
|
241
248
|
# initial detection
|
|
242
|
-
nn_denoiser_max_waveforms_fit: int =
|
|
249
|
+
nn_denoiser_max_waveforms_fit: int = 512_000
|
|
250
|
+
nn_denoiser_noise_waveforms: int = 100 * 256
|
|
251
|
+
nn_denoiser_extra_kwargs: dict | None = None
|
|
243
252
|
do_tpca_denoise: bool = True
|
|
244
253
|
first_denoiser_thinning: float = 0.0
|
|
245
254
|
first_denoiser_spatial_dedup_radius: float = 100.0
|
|
@@ -295,8 +304,8 @@ class DeveloperConfig(DARTsortUserConfig):
|
|
|
295
304
|
initial_pc_scale: float = 2.0
|
|
296
305
|
initial_pc_pre_scale: float = 0.5
|
|
297
306
|
motion_aware_clustering: bool = True
|
|
298
|
-
clustering_max_spikes: Annotated[int, Field(gt=0)] =
|
|
299
|
-
pre_refinement_merge: bool =
|
|
307
|
+
clustering_max_spikes: Annotated[int, Field(gt=0)] = 1024 * 1000
|
|
308
|
+
pre_refinement_merge: bool = True
|
|
300
309
|
post_refinement_merge: bool = False
|
|
301
310
|
pre_refinement_merge_metric: str = "normeuc"
|
|
302
311
|
pre_refinement_merge_threshold: float = 0.1
|