DARTsort 0.5.5__tar.gz → 0.5.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.5 → dartsort-0.5.6}/PKG-INFO +1 -1
- {dartsort-0.5.5 → dartsort-0.5.6}/docs/main_api.md +1 -1
- {dartsort-0.5.5 → dartsort-0.5.6}/docs/toolkit_api.md +17 -1
- {dartsort-0.5.5 → dartsort-0.5.6}/pyproject.toml +5 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/__init__.py +1 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/clustering.py +21 -1
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/clustering_features.py +7 -3
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/density.py +139 -151
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/kmeans.py +15 -32
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/merge.py +7 -12
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/mixture.py +110 -31
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/config.py +4 -3
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/matching.py +4 -3
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/subtract.py +30 -17
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/templates/postprocess_util.py +8 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/templates/realignment.py +3 -1
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/pipeline.py +4 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/cli_util.py +2 -2
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/data_util.py +3 -26
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/drift_util.py +1 -1
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/internal_config.py +9 -10
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/logging_util.py +26 -14
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/main_util.py +2 -3
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/motion.py +3 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/nn_util.py +2 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/peel_util.py +10 -8
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/preprocess_util.py +2 -2
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/spiketorch.py +16 -3
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/waveform_util.py +0 -40
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/analysis_plots.py +4 -2
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/mixture.py +2 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/scatterplots.py +68 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort.egg-info/SOURCES.txt +1 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_dartsort.py +1 -1
- dartsort-0.5.6/tests/test_density.py +71 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_simkit.py +2 -2
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_templates.py +4 -3
- {dartsort-0.5.5 → dartsort-0.5.6}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/.gitignore +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/README.md +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/docs/index.md +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/environment.yml +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/.keep +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/basic_integration_test.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/ci diag.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/dartsort_dredge_demo.ipynb +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/dartsort_dredge_demo.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/data test.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/dead reckoning.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/denoise np1.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/denoise np2.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/exploratory.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/geom debug.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/hdbscan_clustering_5min.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/hybrid 10.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/hybrid cluster.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/hybrid_11.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/initial training.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/make hybrid.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/mgh.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/np1 feats.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/np2 processing version c.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/paperfigs.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/pdists.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/postcheck.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/ptp vae.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/re clust.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/reg fig loop.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/reg_sim_exp.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/relocating.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/relocating_all.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/restats +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/sortStinger1.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/sortStinger3.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/sortStinger4.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/sortStinger5.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/sortStinger6.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/spike farm.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/ss_ensemble_sim.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/ss_ensemble_sim_drift.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/ss_sim.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/stan diagnostics.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/storytelling c.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/storytelling.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/tests/debug_matching.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/tests/test_agreement.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/tests/test_amplitude_scaling.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/tests/test_batch_optim.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/tests/test_ibme.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/tests/test_mm_initialization.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/tests/test_normxcorr1d.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/tests/test_reassignment.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/tests/test_relocation.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/notebook/thomas.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/setup.cfg +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/agglomerate.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/cluster_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/modes.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/clustering/refine_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/evaluate/analysis.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/evaluate/comparison.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/evaluate/simkit.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/main.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/peel_base.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/peel_lib.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/decollider.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/mixture_classifier.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/transform_base.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/noise_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/vismain.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/conftest.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_alignment.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_clustering.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_config.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_data_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_detect.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_linalg.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_matching.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_modeling.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_subtract.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_threshold.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_transform.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.5 → dartsort-0.5.6}/zensical.toml +0 -0
|
@@ -19,7 +19,7 @@ For details on parameters you should think about before running the sorter, see
|
|
|
19
19
|
|
|
20
20
|
The return value from the `dartsort()` function is a `DARTsortResult` object, which is a dictionary containing spike trains and motion information:
|
|
21
21
|
|
|
22
|
-
::: dartsort.
|
|
22
|
+
::: dartsort.DARTsortResult
|
|
23
23
|
options:
|
|
24
24
|
show_bases: false
|
|
25
25
|
|
|
@@ -17,7 +17,13 @@ In *dartsort* those kind of workflows are called "peelers" (inspired by the iter
|
|
|
17
17
|
All of the peelers accept a featurization pipeline or a [`FeaturizationConfig`](#dartsort.FeaturizationConfig) object which handles the spike featurization; these are discussed [in the next section](#featurization-pipelines).
|
|
18
18
|
|
|
19
19
|
*dartsort* includes high-level functions for running various kinds of peelers and corresponding configuration objects.
|
|
20
|
-
These are
|
|
20
|
+
These are in sections below.
|
|
21
|
+
|
|
22
|
+
### Template matching
|
|
23
|
+
|
|
24
|
+
[`match()`](#dartsort.match) runs template matching from known templates if its `template_data` parameter is set, or else estimating templates from `sorting` using [`estimate_template_library()`](#dartsort.estimate_template_library).
|
|
25
|
+
|
|
26
|
+
It is configured by the [matching_cfg](#dartsort.MatchingConfig) argument.
|
|
21
27
|
|
|
22
28
|
::: dartsort.match
|
|
23
29
|
|
|
@@ -25,18 +31,28 @@ These are:
|
|
|
25
31
|
options:
|
|
26
32
|
show_if_no_docstring: true
|
|
27
33
|
|
|
34
|
+
### Neural-net based collision-cleaned spike detection
|
|
35
|
+
|
|
28
36
|
::: dartsort.subtract
|
|
29
37
|
|
|
30
38
|
::: dartsort.SubtractionConfig
|
|
31
39
|
options:
|
|
32
40
|
show_if_no_docstring: true
|
|
33
41
|
|
|
42
|
+
### Thresholding spike detection
|
|
43
|
+
|
|
34
44
|
::: dartsort.threshold
|
|
35
45
|
|
|
36
46
|
::: dartsort.ThresholdingConfig
|
|
37
47
|
options:
|
|
38
48
|
show_if_no_docstring: true
|
|
39
49
|
|
|
50
|
+
|
|
51
|
+
### Spike extraction and featurization at known event times
|
|
52
|
+
|
|
53
|
+
::: dartsort.grab
|
|
54
|
+
|
|
55
|
+
|
|
40
56
|
## Featurization pipelines
|
|
41
57
|
|
|
42
58
|
Featurization is configured by building a `FeaturizationConfig`.
|
|
@@ -81,6 +81,11 @@ ignore = ["F722"]
|
|
|
81
81
|
[tool.ty.src]
|
|
82
82
|
include = ["src", "tests"]
|
|
83
83
|
|
|
84
|
+
[tool.basedpyright]
|
|
85
|
+
include = ["src", "tests"]
|
|
86
|
+
enableTypeIgnoreComments = true
|
|
87
|
+
typeCheckingMode = "standard"
|
|
88
|
+
|
|
84
89
|
[tool.ty.analysis]
|
|
85
90
|
replace-imports-with-any = ["hdbscan"]
|
|
86
91
|
|
|
@@ -352,9 +352,27 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
352
352
|
recording: BaseRecording | None,
|
|
353
353
|
motion: MotionInfo,
|
|
354
354
|
) -> np.ndarray:
|
|
355
|
+
labels, _ = self._cluster_extra(
|
|
356
|
+
features=features,
|
|
357
|
+
stable_features=stable_features,
|
|
358
|
+
sorting=sorting,
|
|
359
|
+
recording=recording,
|
|
360
|
+
motion=motion,
|
|
361
|
+
)
|
|
362
|
+
return labels
|
|
363
|
+
|
|
364
|
+
def _cluster_extra(
|
|
365
|
+
self,
|
|
366
|
+
features: SimpleMatrixFeatures,
|
|
367
|
+
stable_features: StableWaveformFeatures | None,
|
|
368
|
+
sorting: DARTsortSorting,
|
|
369
|
+
recording: BaseRecording | None,
|
|
370
|
+
motion: MotionInfo,
|
|
371
|
+
) -> tuple[np.ndarray, dict]:
|
|
355
372
|
subsampling, ixs = self.handle_sampling(features)
|
|
356
373
|
X = features.features
|
|
357
374
|
X_fit = X[ixs]
|
|
375
|
+
assert self.computation_cfg is not None
|
|
358
376
|
|
|
359
377
|
if not self.uhdversion:
|
|
360
378
|
res = density.density_peaks(
|
|
@@ -369,6 +387,7 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
369
387
|
outlier_radius=self.outlier_radius,
|
|
370
388
|
outlier_neighbor_count=self.outlier_neighbor_count,
|
|
371
389
|
workers=self.workers,
|
|
390
|
+
device=self.computation_cfg.actual_device(),
|
|
372
391
|
)
|
|
373
392
|
else:
|
|
374
393
|
res = density.density_peaks_fancy(
|
|
@@ -403,13 +422,14 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
403
422
|
ixs, res["labels"], rest, other_labels
|
|
404
423
|
)
|
|
405
424
|
else:
|
|
425
|
+
kdtree = None
|
|
406
426
|
labels = res["labels"]
|
|
407
427
|
|
|
408
428
|
labels = cluster_util.decrumb(
|
|
409
429
|
labels, min_size=self.remove_clusters_smaller_than, in_place=True
|
|
410
430
|
)
|
|
411
431
|
|
|
412
|
-
return labels
|
|
432
|
+
return labels, dict(dpc_res=res, kdtree=kdtree, ixs=ixs)
|
|
413
433
|
|
|
414
434
|
|
|
415
435
|
clustering_strategies["dpc"] = DensityPeaksClusterer
|
|
@@ -7,7 +7,11 @@ from torch import Tensor
|
|
|
7
7
|
|
|
8
8
|
from ..util.data_util import DARTsortSorting
|
|
9
9
|
from ..util.drift_util import get_stable_channels
|
|
10
|
-
from ..util.internal_config import
|
|
10
|
+
from ..util.internal_config import (
|
|
11
|
+
ClusteringFeaturesConfig,
|
|
12
|
+
ComputationConfig,
|
|
13
|
+
default_clustering_features_cfg,
|
|
14
|
+
)
|
|
11
15
|
from ..util.interpolation_util import (
|
|
12
16
|
SpikeNeighborhoods,
|
|
13
17
|
StableFeaturesInterpolator,
|
|
@@ -63,8 +67,8 @@ class SimpleMatrixFeatures:
|
|
|
63
67
|
*,
|
|
64
68
|
sorting: DARTsortSorting,
|
|
65
69
|
motion: MotionInfo,
|
|
66
|
-
clustering_features_cfg: ClusteringFeaturesConfig,
|
|
67
|
-
computation_cfg: ComputationConfig | None,
|
|
70
|
+
clustering_features_cfg: ClusteringFeaturesConfig = default_clustering_features_cfg,
|
|
71
|
+
computation_cfg: ComputationConfig | None = None,
|
|
68
72
|
) -> Self:
|
|
69
73
|
computation_cfg = ensure_computation_config(computation_cfg)
|
|
70
74
|
t_s = sorting.times_seconds
|
|
@@ -1,21 +1,16 @@
|
|
|
1
1
|
from threading import local
|
|
2
|
-
from typing import
|
|
2
|
+
from typing import cast
|
|
3
3
|
|
|
4
4
|
import numba
|
|
5
5
|
import numpy as np
|
|
6
6
|
import torch
|
|
7
|
-
import torch.nn.functional as F
|
|
8
7
|
from scipy.sparse import coo_array
|
|
9
8
|
from scipy.sparse.csgraph import connected_components
|
|
10
9
|
from scipy.spatial import KDTree
|
|
11
|
-
from scipy.spatial.distance import pdist, squareform
|
|
12
|
-
from torch import Tensor
|
|
13
10
|
|
|
14
|
-
from ..util.
|
|
15
|
-
from ..util.job_util import ensure_computation_config
|
|
16
|
-
from ..util.logging_util import get_logger, progbar, progrange
|
|
11
|
+
from ..util.logging_util import DARTSORTVERBOSE, get_logger, progbar, progrange
|
|
17
12
|
from ..util.multiprocessing_util import get_pool
|
|
18
|
-
from ..util.
|
|
13
|
+
from ..util.spiketorch import sqeuc_cdist_known_norm
|
|
19
14
|
from .cluster_util import decrumb
|
|
20
15
|
|
|
21
16
|
logger = get_logger(__name__)
|
|
@@ -266,136 +261,118 @@ def remove_border_points(
|
|
|
266
261
|
return new_labels
|
|
267
262
|
|
|
268
263
|
|
|
269
|
-
def
|
|
270
|
-
X,
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
outlier_sigma=3.0,
|
|
274
|
-
kdtree=None,
|
|
275
|
-
workers=1,
|
|
276
|
-
):
|
|
277
|
-
"""Use a KDE to guess the highest density point."""
|
|
278
|
-
n = len(X)
|
|
279
|
-
sigma0 = np.sqrt(X.var(axis=0).sum())
|
|
280
|
-
inliers, kdtree = kdtree_inliers(
|
|
281
|
-
X,
|
|
282
|
-
kdtree=kdtree,
|
|
283
|
-
n_neighbors=outlier_neighbor_count,
|
|
284
|
-
distance_upper_bound=outlier_sigma * sigma0,
|
|
285
|
-
workers=workers,
|
|
286
|
-
)
|
|
287
|
-
|
|
288
|
-
if sigma == "rule_of_thumb":
|
|
289
|
-
sigma_dens = (
|
|
290
|
-
1.06
|
|
291
|
-
* np.linalg.norm(np.std(X[inliers], axis=0))
|
|
292
|
-
* np.power(inliers.sum(), -0.2)
|
|
293
|
-
)
|
|
294
|
-
else:
|
|
295
|
-
sigma_dens = sigma
|
|
296
|
-
|
|
297
|
-
density = get_smoothed_density(X, inliers=inliers, sigma=sigma_dens)
|
|
298
|
-
assert isinstance(density, np.ndarray)
|
|
299
|
-
assert density.shape == (n,)
|
|
300
|
-
|
|
301
|
-
return np.argmax(density)
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
def bucket_density_ratio(
|
|
305
|
-
X: np.ndarray,
|
|
306
|
-
scaled_geom: np.ndarray,
|
|
307
|
-
sigma: float,
|
|
308
|
-
sigma_regional: float,
|
|
264
|
+
def sort_density(
|
|
265
|
+
X: np.ndarray | torch.Tensor,
|
|
266
|
+
sigma0: float,
|
|
267
|
+
sigma1: float,
|
|
309
268
|
max_sigma: float = 3.0,
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
269
|
+
device: torch.device = torch.device("cpu"),
|
|
270
|
+
batch_size: int = 2048,
|
|
271
|
+
col_batch_size: int = 1024,
|
|
272
|
+
show_progress: bool = False,
|
|
273
|
+
) -> np.ndarray:
|
|
274
|
+
"""Compute a ratio of Gaussian KDEs by ordering along the first dimension
|
|
315
275
|
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
scale matches the clustering feature's x scale.)
|
|
319
|
-
- Let Dc be pairwise distances between channels
|
|
320
|
-
- Compute sparse distance matrix. If Dx is pairwise distance between
|
|
321
|
-
all pairs of spikes, we just want the entries with Dx<max_dist.
|
|
322
|
-
- Note
|
|
323
|
-
|ci-cj| = |(ci-xi) + (xj-cj) + (xi-xj)|
|
|
324
|
-
<= 2C + |xi-xj|
|
|
325
|
-
Thus we need not consider i,j s.t. |ci-cj|-2C>max_dist.
|
|
326
|
-
"""
|
|
327
|
-
computation_cfg = ensure_computation_config(computation_cfg)
|
|
328
|
-
with timer("geom bucketing"):
|
|
329
|
-
geom_kdt = KDTree(scaled_geom, leafsize=1)
|
|
330
|
-
dists, channels = geom_kdt.query(X[:, :2], workers=workers)
|
|
331
|
-
|
|
332
|
-
max_dist_by_chan = np.zeros(len(scaled_geom))
|
|
333
|
-
for c in range(len(scaled_geom)):
|
|
334
|
-
inc = np.flatnonzero(channels == c)
|
|
335
|
-
if inc.size:
|
|
336
|
-
max_dist_by_chan[c] = dists[inc].max()
|
|
337
|
-
|
|
338
|
-
larger_sigma = max(sigma, sigma_regional or 0.0)
|
|
339
|
-
max_dist = max_sigma * larger_sigma
|
|
340
|
-
|
|
341
|
-
lb = squareform(pdist(scaled_geom))
|
|
342
|
-
lb -= max_dist_by_chan[:, None]
|
|
343
|
-
lb -= max_dist_by_chan
|
|
344
|
-
chans_pair_mask = lb <= max_dist
|
|
345
|
-
|
|
346
|
-
device = computation_cfg.actual_device()
|
|
347
|
-
channels = torch.asarray(channels, device=device, dtype=torch.long)
|
|
348
|
-
Xt = torch.asarray(X, device=device, dtype=torch.float)
|
|
349
|
-
chans_pair_mask = torch.asarray(chans_pair_mask, device=device, dtype=torch.bool)
|
|
350
|
-
density_ratio = torch.full((len(X),), -torch.inf, device=device)
|
|
351
|
-
|
|
352
|
-
# for c in trange(len(scaled_geom), desc="bktdens"):
|
|
353
|
-
for c in progrange(25, desc="bktdens"):
|
|
354
|
-
(inc,) = (channels == c).nonzero(as_tuple=True)
|
|
355
|
-
(friends,) = chans_pair_mask[c][channels].nonzero(as_tuple=True)
|
|
356
|
-
density_ratio[inc] = _local_sparse_dens_ratio(
|
|
357
|
-
Xt, inc, friends, max_dist, sigma, sigma_regional, batch_size=batch_size
|
|
358
|
-
)
|
|
276
|
+
This is only useful if the first dimension is much larger than the rest.
|
|
277
|
+
Otherwise, hopefully n is not too big ;).
|
|
359
278
|
|
|
360
|
-
|
|
279
|
+
Parameters
|
|
280
|
+
----------
|
|
281
|
+
X : np.ndarray | torch.Tensor
|
|
282
|
+
Data whose density ratio we want.
|
|
283
|
+
sigma0 : float
|
|
284
|
+
The smaller sigma
|
|
285
|
+
sigma1 : float
|
|
286
|
+
The larger sigma
|
|
287
|
+
max_sigma : float, optional
|
|
288
|
+
The maximum sigma value to use. Default is 3.0.
|
|
289
|
+
device : torch.device, optional
|
|
290
|
+
batch_size : int, optional
|
|
291
|
+
show_progress : bool, optional
|
|
361
292
|
|
|
293
|
+
Returns
|
|
294
|
+
-------
|
|
295
|
+
np.ndarray
|
|
296
|
+
The density ratio for each point in X.
|
|
297
|
+
"""
|
|
298
|
+
X = torch.asarray(X, device=device)
|
|
299
|
+
order = torch.argsort(X[:, 0])
|
|
300
|
+
X = X[order]
|
|
301
|
+
X0 = X[:, 0].contiguous()
|
|
302
|
+
Xnormsq = torch.linalg.vector_norm(X, dim=1).square_()
|
|
303
|
+
n, dim = X.shape
|
|
362
304
|
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
X
|
|
366
|
-
inc: Tensor,
|
|
367
|
-
inf: Tensor,
|
|
368
|
-
max_dist: float,
|
|
369
|
-
sigma0: float,
|
|
370
|
-
sigma1: float,
|
|
371
|
-
batch_size: int = 1024,
|
|
372
|
-
):
|
|
373
|
-
nc = inc.shape[0]
|
|
374
|
-
nf = inf.shape[0]
|
|
375
|
-
dens_ratio = X.new_empty(nc)
|
|
305
|
+
radius = max_sigma * max(sigma0, sigma1)
|
|
306
|
+
radius_sq = X.new_tensor(radius * radius)
|
|
307
|
+
dens = X.new_zeros(n)
|
|
376
308
|
|
|
377
|
-
|
|
378
|
-
|
|
309
|
+
k0 = X.new_tensor(-0.5 * sigma0**-2)
|
|
310
|
+
k1 = X.new_tensor(-0.5 * sigma1**-2)
|
|
379
311
|
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
Xc = X[inc[ic0:ic1]]
|
|
383
|
-
for if0 in range(0, nf, batch_size):
|
|
384
|
-
if1 = min(nf, if0 + batch_size)
|
|
385
|
-
Xf = X[inf[if0:if1]]
|
|
312
|
+
n0 = X.new_tensor(-0.5 * dim * np.log(2.0 * np.pi * sigma0 * sigma0))
|
|
313
|
+
n1 = X.new_tensor(-0.5 * dim * np.log(2.0 * np.pi * sigma1 * sigma1))
|
|
386
314
|
|
|
387
|
-
|
|
388
|
-
|
|
315
|
+
if show_progress:
|
|
316
|
+
iter = progrange(0, n, batch_size, desc=f'SortDens:{device.type}')
|
|
317
|
+
else:
|
|
318
|
+
iter = range(0, n, batch_size)
|
|
319
|
+
|
|
320
|
+
i0 = i1 = 0
|
|
321
|
+
dist_buf = X.new_zeros((batch_size * col_batch_size,))
|
|
322
|
+
dist_buf2 = X.new_zeros((batch_size * col_batch_size,))
|
|
323
|
+
dist_buf3 = X.new_zeros((batch_size * col_batch_size,))
|
|
324
|
+
dens0_buf = X.new_zeros((batch_size,))
|
|
325
|
+
dens1_buf = X.new_zeros((batch_size,))
|
|
326
|
+
for q0 in iter:
|
|
327
|
+
q1 = min(n, q0 + batch_size)
|
|
328
|
+
nq = q1 - q0
|
|
329
|
+
|
|
330
|
+
Q = X[q0:q1]
|
|
331
|
+
x0 = Q[0, 0] - radius
|
|
332
|
+
x1 = Q[-1, 0] + radius
|
|
333
|
+
|
|
334
|
+
if i0 == q0:
|
|
335
|
+
assert q0 == 0
|
|
336
|
+
else:
|
|
337
|
+
assert q0 > i0
|
|
338
|
+
i0 = i0 + torch.searchsorted(X0[i0:q0], x0, side="right") - 1
|
|
339
|
+
i0.clamp_(min=0)
|
|
340
|
+
i0 = int(i0.item())
|
|
389
341
|
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
342
|
+
if q1 == n:
|
|
343
|
+
i1 = n
|
|
344
|
+
else:
|
|
345
|
+
i1 = q1 + torch.searchsorted(X0[q1:], x1, side="right")
|
|
346
|
+
i1.clamp_(max=n)
|
|
347
|
+
i1 = int(i1.item())
|
|
348
|
+
|
|
349
|
+
dens0 = dens0_buf[: q1 - q0].zero_()
|
|
350
|
+
dens1 = dens1_buf[: q1 - q0].zero_()
|
|
351
|
+
for j0 in range(i0, i1, col_batch_size):
|
|
352
|
+
j1 = min(j0 + col_batch_size, i1)
|
|
353
|
+
nj = j1 - j0
|
|
354
|
+
nbuf = nq * nj
|
|
355
|
+
|
|
356
|
+
dist = sqeuc_cdist_known_norm(
|
|
357
|
+
X=Q,
|
|
358
|
+
Xnormsq=Xnormsq[q0:q1],
|
|
359
|
+
Y=X[j0:j1],
|
|
360
|
+
Ynormsq=Xnormsq[j0:j1],
|
|
361
|
+
out=dist_buf[:nbuf].view(nq, nj),
|
|
362
|
+
)
|
|
363
|
+
dist.masked_fill_(dist > radius_sq, torch.inf)
|
|
364
|
+
gauss0 = torch.addcmul(
|
|
365
|
+
n0, k0, dist, out=dist_buf2[:nbuf].view(nq, nj)
|
|
366
|
+
).exp_()
|
|
367
|
+
gauss1 = torch.addcmul(
|
|
368
|
+
n1, k1, dist, out=dist_buf3[:nbuf].view(nq, nj)
|
|
369
|
+
).exp_()
|
|
370
|
+
dens0 += gauss0.sum(dim=1)
|
|
371
|
+
dens1 += gauss1.sum(dim=1)
|
|
372
|
+
dens[q0:q1] = dens0.div_(dens1)
|
|
397
373
|
|
|
398
|
-
|
|
374
|
+
dens = dens[torch.argsort(order)]
|
|
375
|
+
return dens.numpy(force=True)
|
|
399
376
|
|
|
400
377
|
|
|
401
378
|
def kdt_density(
|
|
@@ -449,27 +426,34 @@ def _kdtdens_job(i0):
|
|
|
449
426
|
jj = sdm["j"]
|
|
450
427
|
v_local = sdm["v"]
|
|
451
428
|
v_regional = sdm["v"].copy()
|
|
452
|
-
|
|
453
|
-
_gausskernel(
|
|
429
|
+
dim = float(X.shape[1])
|
|
430
|
+
_gausskernel(v_local, sigma, dim)
|
|
431
|
+
_gausskernel(v_regional, sigma_regional, dim)
|
|
454
432
|
coo_local = coo_array(
|
|
455
433
|
(v_local, (ii, jj)), shape=(batch_kdt.n, kdtree.n), dtype=v_local.dtype
|
|
456
434
|
)
|
|
457
435
|
coo_regional = coo_array(
|
|
458
436
|
(v_regional, (ii, jj)), shape=(batch_kdt.n, kdtree.n), dtype=v_local.dtype
|
|
459
437
|
)
|
|
460
|
-
|
|
438
|
+
dens = coo_local.sum(axis=1)
|
|
461
439
|
dens_regional = coo_regional.sum(axis=1)
|
|
462
|
-
|
|
463
|
-
|
|
440
|
+
dens /= dens_regional
|
|
441
|
+
|
|
442
|
+
return i0, i1, dens
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
_log_2pi = np.log(2.0 * np.pi)
|
|
464
446
|
|
|
465
447
|
|
|
466
448
|
@numba.jit(
|
|
467
449
|
parallel=False, nopython=True, boundscheck=False, nogil=True, error_model="numpy"
|
|
468
450
|
)
|
|
469
|
-
def _gausskernel(v: np.ndarray, sigma: float):
|
|
470
|
-
|
|
451
|
+
def _gausskernel(v: np.ndarray, sigma: float, dim: float):
|
|
452
|
+
h = np.sqrt(2.0) * sigma
|
|
453
|
+
const = 1.0 / h
|
|
454
|
+
addconst = (dim / 2) * np.log(2.0 * np.pi * sigma * sigma)
|
|
471
455
|
for i in range(v.size):
|
|
472
|
-
v[i] = np.exp(-np.square(v[i] * const))
|
|
456
|
+
v[i] = np.exp(-np.square(v[i] * const) - addconst)
|
|
473
457
|
|
|
474
458
|
|
|
475
459
|
def knn_density(
|
|
@@ -507,8 +491,6 @@ def density_peaks(
|
|
|
507
491
|
kdtree=None,
|
|
508
492
|
density=None,
|
|
509
493
|
knn_k=None,
|
|
510
|
-
use_knn=False,
|
|
511
|
-
use_histograms=False,
|
|
512
494
|
sigma_local=5.0,
|
|
513
495
|
sigma_regional=None,
|
|
514
496
|
outlier_neighbor_count: int | None = 10,
|
|
@@ -522,7 +504,9 @@ def density_peaks(
|
|
|
522
504
|
border_search_neighbors=3,
|
|
523
505
|
inlier_dims=slice(0, 2),
|
|
524
506
|
leafsize=24,
|
|
507
|
+
density_strategy="sort",
|
|
525
508
|
workers=-1,
|
|
509
|
+
device: torch.device = torch.device("cpu"),
|
|
526
510
|
):
|
|
527
511
|
"""Density peaks clustering as described by Rodriguez and Laio, but...
|
|
528
512
|
|
|
@@ -536,6 +520,7 @@ def density_peaks(
|
|
|
536
520
|
radius_search = radius_search * np.sqrt(X.shape[1])
|
|
537
521
|
dim_arange = np.arange(X.shape[1])
|
|
538
522
|
inlier_dim_ixs = dim_arange[inlier_dims]
|
|
523
|
+
|
|
539
524
|
if outlier_radius is not None:
|
|
540
525
|
outlier_radius = outlier_radius * np.sqrt(inlier_dim_ixs.size)
|
|
541
526
|
inliers, kdtree = kdtree_inliers(
|
|
@@ -550,7 +535,8 @@ def density_peaks(
|
|
|
550
535
|
kdtree = KDTree(X)
|
|
551
536
|
|
|
552
537
|
if density is None:
|
|
553
|
-
if
|
|
538
|
+
if density_strategy == "kdt":
|
|
539
|
+
assert sigma_regional is not None
|
|
554
540
|
density = kdt_density(
|
|
555
541
|
kdtree,
|
|
556
542
|
X,
|
|
@@ -558,24 +544,31 @@ def density_peaks(
|
|
|
558
544
|
sigma_regional=sigma_regional,
|
|
559
545
|
n_threads=workers,
|
|
560
546
|
)
|
|
561
|
-
elif
|
|
562
|
-
|
|
563
|
-
|
|
547
|
+
elif density_strategy == "sort":
|
|
548
|
+
assert sigma_regional is not None
|
|
549
|
+
density = sort_density(
|
|
550
|
+
X,
|
|
551
|
+
sigma0=sigma_local,
|
|
552
|
+
sigma1=sigma_regional,
|
|
553
|
+
device=device,
|
|
554
|
+
show_progress=logger.isEnabledFor(DARTSORTVERBOSE),
|
|
564
555
|
)
|
|
565
|
-
elif
|
|
556
|
+
elif density_strategy == "hist":
|
|
566
557
|
sigmas = [sigma_local]
|
|
567
558
|
if sigma_regional is not None:
|
|
568
559
|
sigmas.append(sigma_regional)
|
|
569
560
|
density = get_smoothed_density_ratio(X, inliers=inliers, sigmas=sigmas)
|
|
570
|
-
|
|
561
|
+
elif density_strategy == "knn":
|
|
571
562
|
density = knn_density(
|
|
572
563
|
kdtree,
|
|
573
564
|
X,
|
|
574
|
-
k=
|
|
565
|
+
k=knn_k,
|
|
575
566
|
distance_upper_bound=radius_search,
|
|
576
567
|
workers=workers,
|
|
577
568
|
sigma=sigma_local,
|
|
578
569
|
)
|
|
570
|
+
else:
|
|
571
|
+
raise ValueError(f"Unknown density strategy: {density_strategy}")
|
|
579
572
|
|
|
580
573
|
nhdn = nearest_higher_density_neighbor(
|
|
581
574
|
kdtree,
|
|
@@ -606,12 +599,7 @@ def density_peaks(
|
|
|
606
599
|
if remove_clusters_smaller_than:
|
|
607
600
|
labels = decrumb(labels, min_size=remove_clusters_smaller_than, in_place=True)
|
|
608
601
|
|
|
609
|
-
return dict(
|
|
610
|
-
density=density,
|
|
611
|
-
nhdn=nhdn,
|
|
612
|
-
labels=labels,
|
|
613
|
-
kdtree=kdtree,
|
|
614
|
-
)
|
|
602
|
+
return dict(density=density, nhdn=nhdn, labels=labels, kdtree=kdtree)
|
|
615
603
|
|
|
616
604
|
|
|
617
605
|
def nearest_neighbor_assign(
|
|
@@ -21,7 +21,7 @@ from ..util.sparse_util import (
|
|
|
21
21
|
logsumexp_coo,
|
|
22
22
|
sparse_centroid_distsq,
|
|
23
23
|
)
|
|
24
|
-
from ..util.spiketorch import spawn_torch_rg
|
|
24
|
+
from ..util.spiketorch import spawn_torch_rg, sqeuc_cdist_known_norm
|
|
25
25
|
|
|
26
26
|
logger = get_logger(__name__)
|
|
27
27
|
|
|
@@ -56,22 +56,11 @@ def kmeanspp(
|
|
|
56
56
|
elif kmeanspp_initial == "mean":
|
|
57
57
|
closest = torch.cdist(X, X.mean(0, keepdim=True)).argmax()
|
|
58
58
|
centroid_ixs[0] = closest.item()
|
|
59
|
-
elif kmeanspp_initial == "mode":
|
|
60
|
-
from .density import guess_mode
|
|
61
|
-
|
|
62
|
-
Xm = X
|
|
63
|
-
if Xm.shape[1] > mode_dim:
|
|
64
|
-
q = min(mode_dim + 10, *Xm.shape)
|
|
65
|
-
u, s, v = torch.pca_lowrank(Xm, q=q, niter=7)
|
|
66
|
-
Xm = u[:, :mode_dim].mul_(s[:mode_dim])
|
|
67
|
-
cixs = guess_mode(Xm.numpy(force=True))
|
|
68
|
-
cixs = torch.asarray(cixs, dtype=centroid_ixs.dtype, device=centroid_ixs.device)
|
|
69
|
-
centroid_ixs.copy_(cixs)
|
|
70
59
|
else:
|
|
71
60
|
assert False
|
|
72
61
|
|
|
73
62
|
if Xnormsq is None:
|
|
74
|
-
Xnormsq = torch.
|
|
63
|
+
Xnormsq = torch.linalg.vector_norm(X, dim=1).square_()
|
|
75
64
|
|
|
76
65
|
dists_ = X.new_zeros(len(X))[:, None]
|
|
77
66
|
dists = _sqeuc(X, Xnormsq[:, None], Y=X[centroid_ixs[0]][None], out=dists_)[:, 0]
|
|
@@ -120,12 +109,15 @@ def kmeanspp(
|
|
|
120
109
|
if invalid.all():
|
|
121
110
|
break
|
|
122
111
|
p.masked_fill_(invalid, 0.0)
|
|
123
|
-
|
|
112
|
+
ci = torch.multinomial(p, 1, generator=gen)
|
|
113
|
+
centroid_ixs[j] = ci
|
|
124
114
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
X
|
|
115
|
+
newdists = sqeuc_cdist_known_norm(
|
|
116
|
+
X,
|
|
117
|
+
Xnormsq,
|
|
118
|
+
X[ci],
|
|
119
|
+
Xnormsq[ci],
|
|
120
|
+
out=p[:, None],
|
|
129
121
|
)[:, 0]
|
|
130
122
|
if not skip_assignment:
|
|
131
123
|
closer = newdists < dists
|
|
@@ -143,6 +135,8 @@ def kmeanspp(
|
|
|
143
135
|
else:
|
|
144
136
|
phi = dists.mul_(weights).mean() / weights.mean()
|
|
145
137
|
|
|
138
|
+
dists.relu_()
|
|
139
|
+
|
|
146
140
|
return centroid_ixs, assignments, dists, phi
|
|
147
141
|
|
|
148
142
|
|
|
@@ -379,7 +373,7 @@ def kmeans_inner(
|
|
|
379
373
|
centroids = X[centroid_ixs]
|
|
380
374
|
|
|
381
375
|
if X_normsq is None:
|
|
382
|
-
X_normsq = torch.linalg.
|
|
376
|
+
X_normsq = torch.linalg.vector_norm(X, dim=1).square_()
|
|
383
377
|
assert X_normsq is not None
|
|
384
378
|
X_normsq = X_normsq[:, None]
|
|
385
379
|
dists_out = X.new_empty((X.shape[0], centroids.shape[0]))
|
|
@@ -483,15 +477,6 @@ def _sqeuc(X: Tensor, Xnormsq: Tensor, Y: Tensor, out: Tensor):
|
|
|
483
477
|
return out
|
|
484
478
|
|
|
485
479
|
|
|
486
|
-
@torch.jit.script
|
|
487
|
-
def _sqeuc_ysqknown(
|
|
488
|
-
X: Tensor, Xnormsq: Tensor, Y: Tensor, Ynormsq: Tensor, out: Tensor
|
|
489
|
-
):
|
|
490
|
-
torch.addmm(Xnormsq, X, Y.transpose(0, 1), alpha=-2.0, out=out)
|
|
491
|
-
out.add_(Ynormsq)
|
|
492
|
-
return out
|
|
493
|
-
|
|
494
|
-
|
|
495
480
|
@torch.jit.script
|
|
496
481
|
def _kmeanspp_simple_loop(
|
|
497
482
|
*,
|
|
@@ -506,7 +491,6 @@ def _kmeanspp_simple_loop(
|
|
|
506
491
|
):
|
|
507
492
|
buf = torch.empty_like(p)
|
|
508
493
|
buf_ = buf[:, None]
|
|
509
|
-
Xnormsq = Xnormsq[:, None]
|
|
510
494
|
closer = X.new_zeros(X.shape[:1], dtype=torch.bool)
|
|
511
495
|
cent = torch.zeros_like(X[:1])
|
|
512
496
|
cnormsq = torch.zeros_like(Xnormsq[:1])
|
|
@@ -522,7 +506,7 @@ def _kmeanspp_simple_loop(
|
|
|
522
506
|
|
|
523
507
|
torch.index_select(X, dim=0, index=cj, out=cent)
|
|
524
508
|
torch.index_select(Xnormsq, dim=0, index=cj, out=cnormsq)
|
|
525
|
-
newdists =
|
|
509
|
+
newdists = sqeuc_cdist_known_norm(X, Xnormsq, cent, cnormsq, out=buf_).view(-1)
|
|
526
510
|
|
|
527
511
|
torch.lt(newdists, dists, out=closer)
|
|
528
512
|
assignments.masked_fill_(closer, j)
|
|
@@ -542,7 +526,6 @@ def _kmeanspp_noassign_loop(
|
|
|
542
526
|
):
|
|
543
527
|
buf = torch.empty_like(p)
|
|
544
528
|
buf_ = buf[:, None]
|
|
545
|
-
Xnormsq = Xnormsq[:, None]
|
|
546
529
|
cent = torch.zeros_like(X[:1])
|
|
547
530
|
cnormsq = torch.zeros_like(Xnormsq[:1])
|
|
548
531
|
for j in range(1, centroid_ixs.shape[0]):
|
|
@@ -557,7 +540,7 @@ def _kmeanspp_noassign_loop(
|
|
|
557
540
|
|
|
558
541
|
torch.index_select(X, dim=0, index=cj, out=cent)
|
|
559
542
|
torch.index_select(Xnormsq, dim=0, index=cj, out=cnormsq)
|
|
560
|
-
newdists =
|
|
543
|
+
newdists = sqeuc_cdist_known_norm(X, Xnormsq, cent, cnormsq, out=buf_).view(-1)
|
|
561
544
|
|
|
562
545
|
torch.minimum(dists, newdists, out=dists)
|
|
563
546
|
|