DARTsort 0.5.9__tar.gz → 0.5.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.9 → dartsort-0.5.11}/PKG-INFO +2 -2
- {dartsort-0.5.9 → dartsort-0.5.11}/README.md +1 -1
- {dartsort-0.5.9 → dartsort-0.5.11}/docs/index.md +1 -1
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/agglomerate.py +295 -31
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/cluster_util.py +4 -3
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/clustering.py +78 -1
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/clustering_features.py +37 -12
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/kmeans.py +296 -5
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/mixture.py +274 -65
- dartsort-0.5.11/src/dartsort/clustering/refine_util.py +775 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/config.py +29 -8
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/analysis.py +48 -7
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/comparison.py +43 -5
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/main.py +18 -11
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/grab.py +89 -16
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching.py +24 -6
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/compressed_upsampled.py +12 -1
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/drifty.py +79 -37
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/matching_base.py +7 -2
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/peel_base.py +55 -16
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/peel_lib.py +69 -29
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/reduction_template.py +14 -9
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/subtract.py +59 -10
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/get_templates.py +2 -2
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/postprocess_util.py +88 -32
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/template_util.py +7 -3
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/templates.py +10 -4
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/templib.py +5 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/_multichan_denoiser_kit.py +12 -2
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/amortized_localization.py +5 -5
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/decollider.py +62 -12
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/mixture_classifier.py +4 -1
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/pipeline.py +7 -1
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/temporal_pca.py +39 -14
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/transform_base.py +22 -3
- dartsort-0.5.11/src/dartsort/transform/whiten.py +102 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/cli_util.py +0 -2
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/data_util.py +278 -12
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/internal_config.py +114 -35
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/main_util.py +7 -4
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/noise_util.py +180 -27
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/spiketorch.py +24 -5
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/testing_util/matching_debug_util.py +40 -6
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/analysis_plots.py +166 -5
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/gt.py +39 -3
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/mixture.py +6 -6
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/sorting.py +60 -8
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/unit.py +194 -84
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/unit_comparison.py +12 -6
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/vismain.py +9 -4
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/waveforms.py +1 -1
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/PKG-INFO +2 -2
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_clustering.py +5 -2
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_config.py +2 -1
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_dartsort.py +1 -1
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_kmeans.py +7 -4
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_matching.py +2 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_subtract.py +8 -9
- dartsort-0.5.9/src/dartsort/clustering/refine_util.py +0 -181
- dartsort-0.5.9/src/dartsort/transform/whiten.py +0 -34
- {dartsort-0.5.9 → dartsort-0.5.11}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/.gitignore +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/docs/main_api.md +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/environment.yml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/.keep +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/basic_integration_test.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/ci diag.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/dartsort_dredge_demo.ipynb +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/dartsort_dredge_demo.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/data test.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/dead reckoning.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/denoise np1.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/denoise np2.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/exploratory.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/geom debug.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/hdbscan_clustering_5min.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/hybrid 10.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/hybrid cluster.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/hybrid_11.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/initial training.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/make hybrid.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/mgh.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/np1 feats.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/np2 processing version c.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/paperfigs.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/pdists.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/postcheck.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/ptp vae.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/re clust.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/reg fig loop.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/reg_sim_exp.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/relocating.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/relocating_all.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/restats +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/sortStinger1.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/sortStinger3.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/sortStinger4.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/sortStinger5.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/sortStinger6.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/spike farm.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/ss_ensemble_sim.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/ss_ensemble_sim_drift.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/ss_sim.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/stan diagnostics.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/storytelling c.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/storytelling.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/debug_matching.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_agreement.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_amplitude_scaling.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_batch_optim.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_ibme.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_mm_initialization.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_normxcorr1d.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_reassignment.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/tests/test_relocation.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/notebook/thomas.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/pyproject.toml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/setup.cfg +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/clustering/modes.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/simkit.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/motion.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/SOURCES.txt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/conftest.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_alignment.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_data_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_density.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_detect.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_linalg.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_modeling.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_simkit.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_templates.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_threshold.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_transform.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.11}/zensical.toml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dartsort
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.11
|
|
4
4
|
Summary: DARTsort
|
|
5
5
|
Author-email: Charlie Windolf <ciw2048@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/cwindolf/dartsort
|
|
@@ -75,7 +75,7 @@ $ pip install dartsort
|
|
|
75
75
|
|
|
76
76
|
If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
|
|
77
77
|
|
|
78
|
-
|
|
78
|
+
### Setting up a Python environment
|
|
79
79
|
|
|
80
80
|
If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
|
|
81
81
|
|
|
@@ -32,7 +32,7 @@ $ pip install dartsort
|
|
|
32
32
|
|
|
33
33
|
If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
### Setting up a Python environment
|
|
36
36
|
|
|
37
37
|
If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
|
|
38
38
|
|
|
@@ -32,7 +32,7 @@ $ pip install dartsort
|
|
|
32
32
|
|
|
33
33
|
If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
### Setting up a Python environment
|
|
36
36
|
|
|
37
37
|
If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
|
|
38
38
|
|
|
@@ -39,7 +39,6 @@ from .cluster_util import (
|
|
|
39
39
|
reorder_by_depth,
|
|
40
40
|
sparsify_labels,
|
|
41
41
|
)
|
|
42
|
-
from .mixture import Scores
|
|
43
42
|
|
|
44
43
|
logger = get_logger(__name__)
|
|
45
44
|
|
|
@@ -56,7 +55,7 @@ class Agglomeration:
|
|
|
56
55
|
def agglomerate(
|
|
57
56
|
*,
|
|
58
57
|
sorting: DARTsortSorting,
|
|
59
|
-
recording: BaseRecording
|
|
58
|
+
recording: BaseRecording,
|
|
60
59
|
template_merge_cfg: TemplateMergeConfig | None,
|
|
61
60
|
refinement_cfg: RefinementConfig | None,
|
|
62
61
|
motion: MotionInfo,
|
|
@@ -104,7 +103,7 @@ def agglomerate(
|
|
|
104
103
|
)
|
|
105
104
|
|
|
106
105
|
# tdist tells us the possible merges
|
|
107
|
-
|
|
106
|
+
distance_mask = linkage_mask(
|
|
108
107
|
tdist.distances,
|
|
109
108
|
linkage_method=template_merge_cfg.linkage,
|
|
110
109
|
threshold=template_merge_cfg.merge_distance_threshold,
|
|
@@ -117,9 +116,9 @@ def agglomerate(
|
|
|
117
116
|
dt=refinement_cfg.glom_firing_corr_dt,
|
|
118
117
|
method=refinement_cfg.glom_firing_corr_method,
|
|
119
118
|
)
|
|
120
|
-
_oldsum =
|
|
119
|
+
_oldsum = distance_mask[np.triu_indices_from(distance_mask)].sum()
|
|
121
120
|
fcorr_mask = fcorr <= refinement_cfg.glom_max_firing_corr
|
|
122
|
-
mask = np.logical_and(
|
|
121
|
+
mask = np.logical_and(distance_mask, fcorr_mask)
|
|
123
122
|
np.fill_diagonal(mask, True)
|
|
124
123
|
_newsum = mask[np.triu_indices_from(mask)].sum()
|
|
125
124
|
logger.dartsortdebug(
|
|
@@ -127,6 +126,7 @@ def agglomerate(
|
|
|
127
126
|
)
|
|
128
127
|
else:
|
|
129
128
|
fcorr = fcorr_mask = None
|
|
129
|
+
mask = distance_mask
|
|
130
130
|
|
|
131
131
|
# restrict mask by overlap criteria
|
|
132
132
|
qda_res = qda(
|
|
@@ -158,22 +158,51 @@ def agglomerate(
|
|
|
158
158
|
assert np.all(qda_mask <= mask)
|
|
159
159
|
if fcorr_mask is not None:
|
|
160
160
|
assert np.all(np.logical_and(qda_mask, fcorr_mask) <= mask)
|
|
161
|
+
|
|
162
|
+
simg = refinement_cfg.spikeinterface_merge_preset
|
|
163
|
+
if simg is not None and simg != "none":
|
|
164
|
+
pair_mask = tdist.distances < refinement_cfg.spikeinterface_merge_max_distance
|
|
165
|
+
if refinement_cfg.spikeinterface_merge_min_coentropy is not None:
|
|
166
|
+
cmask, _ = coentropy_merge_mask(
|
|
167
|
+
sorting=sorting,
|
|
168
|
+
min_coentropy=refinement_cfg.spikeinterface_merge_min_coentropy,
|
|
169
|
+
coverage_threshold=refinement_cfg.spikeinterface_merge_coent_coverage,
|
|
170
|
+
iou_threshold=refinement_cfg.spikeinterface_merge_coent_iou,
|
|
171
|
+
)
|
|
172
|
+
pair_mask = np.logical_or(cmask, pair_mask)
|
|
173
|
+
|
|
174
|
+
si_mask = spikeinterface_merge_mask(
|
|
175
|
+
recording=recording,
|
|
176
|
+
sorting=sorting,
|
|
177
|
+
preset=refinement_cfg.spikeinterface_merge_preset,
|
|
178
|
+
censor_ms=refinement_cfg.censor_ms,
|
|
179
|
+
template_data=tdist.template_data,
|
|
180
|
+
pair_mask=pair_mask,
|
|
181
|
+
)
|
|
182
|
+
else:
|
|
183
|
+
si_mask = None
|
|
184
|
+
|
|
185
|
+
# force merges for very close neighbors
|
|
161
186
|
force_mask = linkage_mask(
|
|
162
187
|
tdist.distances,
|
|
163
188
|
linkage_method=template_merge_cfg.linkage,
|
|
164
189
|
threshold=refinement_cfg.qda_force_merge_for_temp_dist_below,
|
|
165
190
|
)
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
191
|
+
|
|
192
|
+
# extract final mask
|
|
193
|
+
final_mask = np.logical_or(qda_mask, force_mask)
|
|
194
|
+
if si_mask is not None:
|
|
195
|
+
final_mask = np.logical_or(final_mask, si_mask)
|
|
196
|
+
np.fill_diagonal(final_mask, True)
|
|
197
|
+
final_mask_as_distance = np.logical_not(final_mask).astype(np.float32)
|
|
169
198
|
|
|
170
199
|
agg_sorting, new_ids = recluster(
|
|
171
200
|
sorting=sorting,
|
|
172
201
|
unit_ids=tdist.template_data.unit_ids,
|
|
173
|
-
dists=
|
|
202
|
+
dists=final_mask_as_distance,
|
|
174
203
|
shifts=tdist.shifts,
|
|
175
204
|
unit_snrs=tdist.template_data.snrs_by_channel().max(1),
|
|
176
|
-
threshold=0.5,
|
|
205
|
+
threshold=0.5, # binary input here
|
|
177
206
|
link=template_merge_cfg.linkage,
|
|
178
207
|
)
|
|
179
208
|
|
|
@@ -331,25 +360,103 @@ def template_distances(
|
|
|
331
360
|
)
|
|
332
361
|
|
|
333
362
|
|
|
334
|
-
def
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
363
|
+
def spikeinterface_merge_mask(
|
|
364
|
+
*,
|
|
365
|
+
recording: BaseRecording,
|
|
366
|
+
sorting: DARTsortSorting,
|
|
367
|
+
preset: str | None,
|
|
368
|
+
censor_ms: float = 0.0,
|
|
369
|
+
template_data: TemplateData,
|
|
370
|
+
pair_mask: np.ndarray,
|
|
371
|
+
min_count: int = 100,
|
|
372
|
+
):
|
|
373
|
+
from spikeinterface.curation.auto_merge import compute_merge_unit_groups
|
|
374
|
+
from spikeinterface.postprocessing import ComputeTemplateSimilarity
|
|
338
375
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
376
|
+
# censor first
|
|
377
|
+
if censor_ms:
|
|
378
|
+
sorting = deduplicate_spikes(sorting, censor_ms)
|
|
342
379
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
380
|
+
# analyzer (lightweight one)
|
|
381
|
+
analyzer = sorting.to_sorting_analyzer(
|
|
382
|
+
recording=recording,
|
|
383
|
+
template_data=template_data,
|
|
384
|
+
compute_extensions=None,
|
|
385
|
+
compute_extensions_if_templates=None,
|
|
386
|
+
estimate_si_sparsity=False,
|
|
387
|
+
compute_template_similarity=False,
|
|
388
|
+
)
|
|
346
389
|
|
|
347
|
-
|
|
348
|
-
|
|
390
|
+
# register the mask as the template similarity extension
|
|
391
|
+
tsim_ext = ComputeTemplateSimilarity(analyzer)
|
|
392
|
+
tsim_ext.data = {"similarity": pair_mask.astype(np.float32)}
|
|
393
|
+
tsim_ext.params = {"method": "dartsort"}
|
|
394
|
+
tsim_ext.run_info = {"run_completed": True}
|
|
395
|
+
analyzer.extensions["template_similarity"] = tsim_ext
|
|
396
|
+
|
|
397
|
+
# handle custom presets
|
|
398
|
+
if preset == "dartsort_slay_xc":
|
|
399
|
+
steps = [
|
|
400
|
+
"num_spikes",
|
|
401
|
+
"remove_contaminated",
|
|
402
|
+
"unit_locations",
|
|
403
|
+
"template_similarity",
|
|
404
|
+
"slay_score",
|
|
405
|
+
"cross_contamination",
|
|
406
|
+
"quality_score",
|
|
407
|
+
]
|
|
408
|
+
preset = None
|
|
409
|
+
analyzer.compute_one_extension("correlograms")
|
|
410
|
+
elif preset == "dartsort_slay_ccg":
|
|
411
|
+
steps = [
|
|
412
|
+
"num_spikes",
|
|
413
|
+
"remove_contaminated",
|
|
414
|
+
"unit_locations",
|
|
415
|
+
"template_similarity",
|
|
416
|
+
"correlogram",
|
|
417
|
+
"slay_score",
|
|
418
|
+
"quality_score",
|
|
419
|
+
]
|
|
420
|
+
preset = None
|
|
421
|
+
analyzer.compute_one_extension("correlograms")
|
|
422
|
+
elif preset == "dartsort_slay_xc_ccg":
|
|
423
|
+
steps = [
|
|
424
|
+
"num_spikes",
|
|
425
|
+
"remove_contaminated",
|
|
426
|
+
"unit_locations",
|
|
427
|
+
"template_similarity",
|
|
428
|
+
"correlogram",
|
|
429
|
+
"cross_contamination",
|
|
430
|
+
"slay_score",
|
|
431
|
+
"quality_score",
|
|
432
|
+
]
|
|
433
|
+
preset = None
|
|
434
|
+
analyzer.compute_one_extension("correlograms")
|
|
435
|
+
else:
|
|
436
|
+
assert preset is not None
|
|
437
|
+
steps = None
|
|
438
|
+
|
|
439
|
+
# make parameters aware of censorship and other params
|
|
440
|
+
my_step_params = {
|
|
441
|
+
"num_spikes": {"min_spikes": min_count},
|
|
442
|
+
"remove_contaminated": {"censored_period_ms": censor_ms},
|
|
443
|
+
"template_similarity": {"similarity_method": "dartsort"},
|
|
444
|
+
"correlogram": {"censor_correlograms_ms": censor_ms},
|
|
445
|
+
"cross_contamination": {"censored_period_ms": censor_ms},
|
|
446
|
+
"quality_score": {"censored_period_ms": censor_ms},
|
|
447
|
+
}
|
|
448
|
+
groups = compute_merge_unit_groups(
|
|
449
|
+
preset=preset,
|
|
450
|
+
steps=steps,
|
|
451
|
+
sorting_analyzer=analyzer,
|
|
452
|
+
steps_params=my_step_params,
|
|
453
|
+
force_copy=False,
|
|
349
454
|
)
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
455
|
+
mask = np.zeros_like(pair_mask)
|
|
456
|
+
for g in groups:
|
|
457
|
+
g = np.array(g)
|
|
458
|
+
mask[g[:, None], g[None, :]] = True
|
|
459
|
+
return mask
|
|
353
460
|
|
|
354
461
|
|
|
355
462
|
@databag
|
|
@@ -392,8 +499,12 @@ def qda(
|
|
|
392
499
|
show_progress: bool,
|
|
393
500
|
computation_cfg: ComputationConfig,
|
|
394
501
|
) -> QDAResult:
|
|
502
|
+
from ..util.data_util import get_gmm_scores
|
|
503
|
+
|
|
395
504
|
# reconstruct scores from sorting attached data (exclude train_ix?)
|
|
396
|
-
|
|
505
|
+
gscores = get_gmm_scores(sorting)
|
|
506
|
+
glabels = sorting.labels
|
|
507
|
+
assert glabels is not None
|
|
397
508
|
|
|
398
509
|
if mask is None:
|
|
399
510
|
mask = np.ones((sorting.n_units, sorting.n_units), dtype=bool)
|
|
@@ -665,7 +776,8 @@ def combine_gmm_scores(
|
|
|
665
776
|
|
|
666
777
|
# check invariants at the top
|
|
667
778
|
if responsibilities.shape[1] > 2:
|
|
668
|
-
|
|
779
|
+
_maxdiff = np.diff(responsibilities[:, :-1], axis=1).max()
|
|
780
|
+
assert _maxdiff <= 1e-3, _maxdiff
|
|
669
781
|
assert np.greater_equal(np.isneginf(logliks[:, :-1]), candidates == -1).all()
|
|
670
782
|
if sorting.labels is not None:
|
|
671
783
|
assert np.all(
|
|
@@ -698,7 +810,8 @@ def combine_gmm_scores(
|
|
|
698
810
|
|
|
699
811
|
# check invariants at the bottom
|
|
700
812
|
if mergedr.shape[1] > 2:
|
|
701
|
-
|
|
813
|
+
_maxdiff = np.diff(mergedr[:, : cand.shape[1]], axis=1).max()
|
|
814
|
+
assert _maxdiff <= 1e-3, _maxdiff
|
|
702
815
|
assert np.greater_equal(np.isneginf(mergedl[:, : cand.shape[1]]), cand == -1).all()
|
|
703
816
|
assert (cand < 0).sum() >= nbye
|
|
704
817
|
if sorting.labels is not None:
|
|
@@ -733,7 +846,7 @@ def _combine_loop(
|
|
|
733
846
|
continue
|
|
734
847
|
|
|
735
848
|
eq_ncandj = rcand[j + 1 :] == ncandj
|
|
736
|
-
if eq_ncandj.sum()
|
|
849
|
+
if eq_ncandj.sum() < 1:
|
|
737
850
|
continue
|
|
738
851
|
|
|
739
852
|
rsum = mergedr[s, j]
|
|
@@ -758,7 +871,7 @@ def _combine_loop(
|
|
|
758
871
|
def deduplicate_spikes(
|
|
759
872
|
sorting: DARTsortSorting,
|
|
760
873
|
radius_ms: float = -1.0,
|
|
761
|
-
score_by=("
|
|
874
|
+
score_by=("merged_log_liks", "gmm_log_liks", "scores"),
|
|
762
875
|
) -> DARTsortSorting:
|
|
763
876
|
"""The lower-scoring of any spikes within radius_samples of each other is relabeled to -1.
|
|
764
877
|
|
|
@@ -783,7 +896,7 @@ def deduplicate_spikes(
|
|
|
783
896
|
break
|
|
784
897
|
if scores is None:
|
|
785
898
|
raise ValueError(f"sorting had none of {score_by}.")
|
|
786
|
-
if scores.ndim
|
|
899
|
+
if scores.ndim >= 2:
|
|
787
900
|
scores = scores[:, 0]
|
|
788
901
|
assert scores.ndim == 1
|
|
789
902
|
assert scores.shape == new_labels.shape
|
|
@@ -799,6 +912,8 @@ def deduplicate_spikes(
|
|
|
799
912
|
ndrop = 0
|
|
800
913
|
for unit_id in unit_ids:
|
|
801
914
|
in_unit = np.flatnonzero(new_labels == unit_id)
|
|
915
|
+
if in_unit.size <= 1:
|
|
916
|
+
continue
|
|
802
917
|
t = times_samples[in_unit]
|
|
803
918
|
dt = np.diff(t)
|
|
804
919
|
if dt.min() > radius_samples:
|
|
@@ -869,3 +984,152 @@ def _dedup_unit_loop(
|
|
|
869
984
|
break
|
|
870
985
|
|
|
871
986
|
i0 = i1
|
|
987
|
+
|
|
988
|
+
|
|
989
|
+
@databag
|
|
990
|
+
class CoentropyResult:
|
|
991
|
+
coentropy: np.ndarray
|
|
992
|
+
"""KxK; reduction of entropy per cooccurrence due to merging pair"""
|
|
993
|
+
|
|
994
|
+
cooccurrence: np.ndarray
|
|
995
|
+
"""KxK; number of times these units score the same spike"""
|
|
996
|
+
|
|
997
|
+
rival_count: np.ndarray
|
|
998
|
+
"""KxK; number of times one unit scores a spike where the other is top"""
|
|
999
|
+
|
|
1000
|
+
occurrence: np.ndarray
|
|
1001
|
+
"""K; number of times the unit appears in the candidates at all"""
|
|
1002
|
+
|
|
1003
|
+
cov: np.ndarray
|
|
1004
|
+
"""KxK; rival count / max pair count (rival diag)"""
|
|
1005
|
+
|
|
1006
|
+
iou: np.ndarray
|
|
1007
|
+
"""KxK; rival count over pair sum"""
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
def coentropy_merge_mask(
|
|
1011
|
+
sorting: DARTsortSorting,
|
|
1012
|
+
min_coentropy: float,
|
|
1013
|
+
coverage_threshold: float,
|
|
1014
|
+
iou_threshold: float,
|
|
1015
|
+
gmm_prefix=("merged", "gmm"),
|
|
1016
|
+
) -> tuple[np.ndarray, CoentropyResult]:
|
|
1017
|
+
"""
|
|
1018
|
+
Parameters
|
|
1019
|
+
----------
|
|
1020
|
+
sorting : DARTsortSorting
|
|
1021
|
+
min_coentropy : float
|
|
1022
|
+
Must be met by pair for mask=True
|
|
1023
|
+
min_coverage : float
|
|
1024
|
+
Pairs such that at least one unit in each pair has
|
|
1025
|
+
rival_count/count > mincov are allowed
|
|
1026
|
+
iou_threshold: float
|
|
1027
|
+
Pairs with rival iou > iouthresh are allowed
|
|
1028
|
+
"""
|
|
1029
|
+
c = coentropy(sorting, gmm_prefix=gmm_prefix)
|
|
1030
|
+
assert c is not None
|
|
1031
|
+
|
|
1032
|
+
mask = np.logical_or(c.cov >= coverage_threshold, c.iou >= iou_threshold)
|
|
1033
|
+
mask = np.logical_and(c.coentropy >= min_coentropy, mask)
|
|
1034
|
+
np.fill_diagonal(mask, True)
|
|
1035
|
+
return mask, c
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
def coentropy(
|
|
1039
|
+
sorting: DARTsortSorting,
|
|
1040
|
+
gmm_prefix=("merged", "gmm"),
|
|
1041
|
+
) -> CoentropyResult | None:
|
|
1042
|
+
"""Calculate entropy reduction due to merging pairs."""
|
|
1043
|
+
for k in gmm_prefix:
|
|
1044
|
+
cands = getattr(sorting, f"{k}_candidates", None)
|
|
1045
|
+
resps = getattr(sorting, f"{k}_responsibilities", None)
|
|
1046
|
+
if cands is not None:
|
|
1047
|
+
assert resps is not None
|
|
1048
|
+
break
|
|
1049
|
+
else:
|
|
1050
|
+
return None
|
|
1051
|
+
|
|
1052
|
+
k = sorting.n_units
|
|
1053
|
+
resps = resps[:, : cands.shape[1]].astype(np.float64)
|
|
1054
|
+
coentropy = np.zeros((k, k))
|
|
1055
|
+
cooccurrence = np.zeros((k, k), dtype=np.int64)
|
|
1056
|
+
rival_count = np.zeros((k, k), dtype=np.int64)
|
|
1057
|
+
occurrence = np.zeros((k,), dtype=np.int64)
|
|
1058
|
+
_calc_coentropy(coentropy, cooccurrence, rival_count, occurrence, cands, resps)
|
|
1059
|
+
rival_count += rival_count.T
|
|
1060
|
+
cdiag = np.diagonal(rival_count)
|
|
1061
|
+
assert (cdiag % 2 == 0).all()
|
|
1062
|
+
np.fill_diagonal(rival_count, cdiag // 2)
|
|
1063
|
+
coentropy += coentropy.T
|
|
1064
|
+
cooccurrence += cooccurrence.T
|
|
1065
|
+
|
|
1066
|
+
# rival count diagonal is just unit top count (not exactly label count,
|
|
1067
|
+
# since it doesn't account for noise assignments)
|
|
1068
|
+
counts = np.diagonal(rival_count)
|
|
1069
|
+
counts = np.maximum(counts, 1)
|
|
1070
|
+
|
|
1071
|
+
cov = rival_count / counts
|
|
1072
|
+
cov = np.minimum(cov, cov.T)
|
|
1073
|
+
|
|
1074
|
+
# this is a disjoint union, since it's the top-label count
|
|
1075
|
+
union = counts[:, None] + counts[None, :]
|
|
1076
|
+
iou = rival_count / union
|
|
1077
|
+
|
|
1078
|
+
return CoentropyResult(
|
|
1079
|
+
coentropy=coentropy,
|
|
1080
|
+
cooccurrence=cooccurrence,
|
|
1081
|
+
rival_count=rival_count,
|
|
1082
|
+
occurrence=occurrence,
|
|
1083
|
+
cov=cov,
|
|
1084
|
+
iou=iou,
|
|
1085
|
+
)
|
|
1086
|
+
|
|
1087
|
+
|
|
1088
|
+
@numba.njit(parallel=True)
|
|
1089
|
+
def _calc_coentropy(
|
|
1090
|
+
coentropy: np.ndarray,
|
|
1091
|
+
cooccurrence: np.ndarray,
|
|
1092
|
+
rival_count: np.ndarray,
|
|
1093
|
+
occurrence: np.ndarray,
|
|
1094
|
+
cands: np.ndarray,
|
|
1095
|
+
resps: np.ndarray,
|
|
1096
|
+
):
|
|
1097
|
+
for i in numba.prange(cands.shape[0]): # ty: ignore
|
|
1098
|
+
u = cands[i]
|
|
1099
|
+
q = resps[i]
|
|
1100
|
+
log_q = np.log(q)
|
|
1101
|
+
np.nan_to_num(log_q, copy=False, neginf=0.0)
|
|
1102
|
+
dh = q * log_q
|
|
1103
|
+
|
|
1104
|
+
ui0 = u[0]
|
|
1105
|
+
qi0 = q[0]
|
|
1106
|
+
dhi0 = dh[0]
|
|
1107
|
+
|
|
1108
|
+
occurrence[ui0] += 1
|
|
1109
|
+
rival_count[ui0, ui0] += 1
|
|
1110
|
+
|
|
1111
|
+
for j in range(1, cands.shape[1]):
|
|
1112
|
+
uj = u[j]
|
|
1113
|
+
if uj < 0:
|
|
1114
|
+
break
|
|
1115
|
+
|
|
1116
|
+
ii = min(ui0, uj)
|
|
1117
|
+
jj = max(ui0, uj)
|
|
1118
|
+
|
|
1119
|
+
occurrence[uj] += 1
|
|
1120
|
+
rival_count[ui0, uj] += 1
|
|
1121
|
+
|
|
1122
|
+
cij = cooccurrence[ii, jj] + 1
|
|
1123
|
+
cooccurrence[ii, jj] = cij
|
|
1124
|
+
|
|
1125
|
+
# change in entropy due to merging uj, uk:
|
|
1126
|
+
# subtract their current contribution, add the new contribution
|
|
1127
|
+
# we want reduction of entropy, so this is the negative of that!
|
|
1128
|
+
qij = q[j] + qi0
|
|
1129
|
+
dhij = dh[j] + dhi0
|
|
1130
|
+
if qij > 0:
|
|
1131
|
+
dhij -= qij * np.log(qij)
|
|
1132
|
+
|
|
1133
|
+
# Welford mean of -dh
|
|
1134
|
+
cur_coent = coentropy[ii, jj]
|
|
1135
|
+
coentropy[ii, jj] = cur_coent + (-dhij - cur_coent) / cij
|
|
@@ -170,10 +170,11 @@ def linkage_mask(
|
|
|
170
170
|
return mask
|
|
171
171
|
|
|
172
172
|
|
|
173
|
-
def sparsify_labels(labels: np.ndarray) -> dict[int, np.ndarray]:
|
|
173
|
+
def sparsify_labels(labels: np.ndarray, ids: np.ndarray | None = None) -> dict[int, np.ndarray]:
|
|
174
174
|
assert labels.ndim == 1
|
|
175
|
-
ids
|
|
176
|
-
|
|
175
|
+
if ids is None:
|
|
176
|
+
ids = np.unique(labels)
|
|
177
|
+
ids = ids[ids >= 0]
|
|
177
178
|
inj = {}
|
|
178
179
|
for j in ids:
|
|
179
180
|
inj[j] = np.flatnonzero(labels == j)
|
|
@@ -19,7 +19,16 @@ from ..util.job_util import ensure_computation_config
|
|
|
19
19
|
from ..util.main_util import ds_save_intermediate_labels
|
|
20
20
|
from ..util.motion import MotionInfo
|
|
21
21
|
from ..util.multiprocessing_util import handle_negative_jobs
|
|
22
|
-
from . import
|
|
22
|
+
from ..util.torch_util import cleanup_and_log_gpu_usage
|
|
23
|
+
from . import (
|
|
24
|
+
agglomerate,
|
|
25
|
+
cluster_util,
|
|
26
|
+
density,
|
|
27
|
+
forward_backward,
|
|
28
|
+
kmeans,
|
|
29
|
+
mixture,
|
|
30
|
+
refine_util,
|
|
31
|
+
)
|
|
23
32
|
from .clustering_features import SimpleMatrixFeatures, StableWaveformFeatures
|
|
24
33
|
|
|
25
34
|
if TYPE_CHECKING:
|
|
@@ -292,6 +301,10 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
292
301
|
workers=-1,
|
|
293
302
|
uhdversion=False,
|
|
294
303
|
random_seed=0,
|
|
304
|
+
kmeans_cleanup=True,
|
|
305
|
+
kmeans_max_sigma: float = 5.0,
|
|
306
|
+
kmeans_iter=100,
|
|
307
|
+
device=None,
|
|
295
308
|
**kwargs,
|
|
296
309
|
):
|
|
297
310
|
super().__init__(**kwargs)
|
|
@@ -308,6 +321,10 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
308
321
|
self.workers = workers
|
|
309
322
|
self.uhdversion = uhdversion
|
|
310
323
|
self.random_seed = random_seed
|
|
324
|
+
self.kmeans_cleanup = kmeans_cleanup
|
|
325
|
+
self.kmeans_max_sigma = kmeans_max_sigma
|
|
326
|
+
self.kmeans_iter = kmeans_iter
|
|
327
|
+
self.device = device
|
|
311
328
|
|
|
312
329
|
@classmethod
|
|
313
330
|
def from_config(
|
|
@@ -335,6 +352,7 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
335
352
|
outlier_radius=clustering_cfg.outlier_radius,
|
|
336
353
|
outlier_neighbor_count=clustering_cfg.outlier_neighbor_count,
|
|
337
354
|
workers=workers,
|
|
355
|
+
device=computation_cfg.actual_device(),
|
|
338
356
|
uhdversion=uhdversion,
|
|
339
357
|
computation_cfg=computation_cfg,
|
|
340
358
|
waveform_cfg=waveform_cfg,
|
|
@@ -342,6 +360,9 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
342
360
|
save_labels_dir=save_labels_dir,
|
|
343
361
|
labels_fmt=labels_fmt,
|
|
344
362
|
sampling_cfg=clustering_cfg.sampling_cfg,
|
|
363
|
+
kmeans_cleanup=clustering_cfg.dpc_kmeans_cleanup,
|
|
364
|
+
kmeans_iter=clustering_cfg.kmeans_iter,
|
|
365
|
+
kmeans_max_sigma=clustering_cfg.gmmdpc_max_sigma,
|
|
345
366
|
)
|
|
346
367
|
|
|
347
368
|
def _cluster(
|
|
@@ -425,6 +446,19 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
425
446
|
kdtree = None
|
|
426
447
|
labels = res["labels"]
|
|
427
448
|
|
|
449
|
+
if self.kmeans_cleanup:
|
|
450
|
+
kres = kmeans.truncated_kmeans_from_labels(
|
|
451
|
+
X=X,
|
|
452
|
+
labels=labels,
|
|
453
|
+
device=self.device,
|
|
454
|
+
max_sigma=self.kmeans_max_sigma,
|
|
455
|
+
n_iter=self.kmeans_iter,
|
|
456
|
+
)
|
|
457
|
+
assert kres.labels is not None
|
|
458
|
+
labels = kres.labels.numpy(force=True)
|
|
459
|
+
del kres
|
|
460
|
+
cleanup_and_log_gpu_usage(self.computation_cfg, "DPC->kmeans")
|
|
461
|
+
|
|
428
462
|
labels = cluster_util.decrumb(
|
|
429
463
|
labels, min_size=self.remove_clusters_smaller_than, in_place=True
|
|
430
464
|
)
|
|
@@ -821,6 +855,7 @@ class AgglomerateRefinement(Refinement):
|
|
|
821
855
|
recording: BaseRecording | None,
|
|
822
856
|
motion: MotionInfo,
|
|
823
857
|
):
|
|
858
|
+
assert recording is not None
|
|
824
859
|
return agglomerate.agglomerate(
|
|
825
860
|
recording=recording,
|
|
826
861
|
sorting=sorting,
|
|
@@ -835,6 +870,48 @@ class AgglomerateRefinement(Refinement):
|
|
|
835
870
|
refinement_strategies["agglomerate"] = AgglomerateRefinement
|
|
836
871
|
|
|
837
872
|
|
|
873
|
+
class FilterRefinement(Refinement):
|
|
874
|
+
"""Runs various filters as specified in cfg"""
|
|
875
|
+
|
|
876
|
+
def needs_stable_features(self):
|
|
877
|
+
return super().needs_stable_features() or bool(
|
|
878
|
+
self.refinement_cfg.collision_cleaning_error_threshold
|
|
879
|
+
)
|
|
880
|
+
|
|
881
|
+
def _refine(
|
|
882
|
+
self,
|
|
883
|
+
features: SimpleMatrixFeatures,
|
|
884
|
+
stable_features: StableWaveformFeatures | None,
|
|
885
|
+
sorting: DARTsortSorting,
|
|
886
|
+
recording: BaseRecording | None,
|
|
887
|
+
motion: MotionInfo,
|
|
888
|
+
):
|
|
889
|
+
assert recording is not None
|
|
890
|
+
|
|
891
|
+
if self.refinement_cfg.collision_cleaning_error_threshold:
|
|
892
|
+
assert stable_features is not None
|
|
893
|
+
sorting = refine_util.collision_cleaning_error_filter(
|
|
894
|
+
sorting=sorting,
|
|
895
|
+
recording=recording,
|
|
896
|
+
stable_features=stable_features,
|
|
897
|
+
refinement_cfg=self.refinement_cfg,
|
|
898
|
+
motion=motion,
|
|
899
|
+
computation_cfg=self.computation_cfg,
|
|
900
|
+
).sorting
|
|
901
|
+
|
|
902
|
+
if self.refinement_cfg.gmm_isolation_threshold:
|
|
903
|
+
sorting = refine_util.gmm_isolation_filter(
|
|
904
|
+
sorting=sorting,
|
|
905
|
+
refinement_cfg=self.refinement_cfg,
|
|
906
|
+
computation_cfg=self.computation_cfg,
|
|
907
|
+
).sorting
|
|
908
|
+
|
|
909
|
+
return sorting
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
refinement_strategies["filter"] = FilterRefinement
|
|
913
|
+
|
|
914
|
+
|
|
838
915
|
class ForwardBackwardEnsembler(Refinement):
|
|
839
916
|
"""If there are more time chunk ones, make a new ABC with this logic."""
|
|
840
917
|
|