DARTsort 0.5.8__tar.gz → 0.5.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.8 → dartsort-0.5.10}/PKG-INFO +3 -3
- {dartsort-0.5.8 → dartsort-0.5.10}/README.md +1 -1
- {dartsort-0.5.8 → dartsort-0.5.10}/docs/index.md +1 -1
- {dartsort-0.5.8 → dartsort-0.5.10}/pyproject.toml +1 -1
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/agglomerate.py +290 -15
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/clustering.py +36 -1
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/clustering_features.py +37 -12
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/kmeans.py +309 -11
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/mixture.py +200 -67
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/config.py +13 -6
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/analysis.py +19 -7
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/comparison.py +33 -4
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/main.py +20 -12
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/grab.py +89 -16
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching.py +24 -6
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/compressed_upsampled.py +12 -1
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/drifty.py +82 -40
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/matching_base.py +12 -7
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/peel_base.py +55 -16
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/peel_lib.py +63 -29
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/reduction_template.py +14 -9
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/subtract.py +51 -10
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/get_templates.py +2 -2
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/postprocess_util.py +4 -4
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/realignment.py +2 -1
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/template_util.py +7 -3
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/templates.py +10 -4
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/templib.py +5 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/_multichan_denoiser_kit.py +12 -2
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/decollider.py +62 -12
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/mixture_classifier.py +3 -2
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/pipeline.py +7 -1
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/temporal_pca.py +39 -14
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/transform_base.py +22 -3
- dartsort-0.5.10/src/dartsort/transform/whiten.py +102 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/data_util.py +171 -1
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/internal_config.py +81 -31
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/interpolation_util.py +5 -5
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/main_util.py +6 -4
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/motion.py +6 -4
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/noise_util.py +180 -27
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/spiketorch.py +68 -18
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/testing_util/matching_debug_util.py +40 -6
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/torch_util.py +46 -3
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/analysis_plots.py +166 -5
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/gt.py +0 -3
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/mixture.py +6 -6
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/unit.py +174 -71
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/unit_comparison.py +12 -6
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/PKG-INFO +3 -3
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/requires.txt +1 -1
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_config.py +2 -1
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_kmeans.py +7 -4
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_matching.py +2 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_subtract.py +8 -9
- dartsort-0.5.8/src/dartsort/transform/whiten.py +0 -34
- {dartsort-0.5.8 → dartsort-0.5.10}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/.gitignore +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/docs/main_api.md +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/environment.yml +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/.keep +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/basic_integration_test.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/ci diag.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/dartsort_dredge_demo.ipynb +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/dartsort_dredge_demo.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/data test.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/dead reckoning.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/denoise np1.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/denoise np2.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/exploratory.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/geom debug.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/hdbscan_clustering_5min.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/hybrid 10.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/hybrid cluster.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/hybrid_11.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/initial training.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/make hybrid.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/mgh.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/np1 feats.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/np2 processing version c.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/paperfigs.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/pdists.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/postcheck.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/ptp vae.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/re clust.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/reg fig loop.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/reg_sim_exp.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/relocating.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/relocating_all.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/restats +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/sortStinger1.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/sortStinger3.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/sortStinger4.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/sortStinger5.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/sortStinger6.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/spike farm.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/ss_ensemble_sim.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/ss_ensemble_sim_drift.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/ss_sim.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/stan diagnostics.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/storytelling c.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/storytelling.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/debug_matching.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_agreement.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_amplitude_scaling.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_batch_optim.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_ibme.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_mm_initialization.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_normxcorr1d.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_reassignment.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/tests/test_relocation.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/notebook/thomas.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/setup.cfg +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/cluster_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/modes.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/clustering/refine_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/simkit.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/vismain.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/SOURCES.txt +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/conftest.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_alignment.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_clustering.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_data_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_density.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_detect.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_linalg.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_modeling.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_simkit.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_templates.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_threshold.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_transform.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.8 → dartsort-0.5.10}/zensical.toml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dartsort
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.10
|
|
4
4
|
Summary: DARTsort
|
|
5
5
|
Author-email: Charlie Windolf <ciw2048@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/cwindolf/dartsort
|
|
@@ -27,7 +27,7 @@ Requires-Dist: scipy>=1.13
|
|
|
27
27
|
Requires-Dist: scikit-learn
|
|
28
28
|
Requires-Dist: spikeinterface>=0.104.0
|
|
29
29
|
Requires-Dist: sympy
|
|
30
|
-
Requires-Dist: torch>=2.
|
|
30
|
+
Requires-Dist: torch>=2.2
|
|
31
31
|
Requires-Dist: tqdm
|
|
32
32
|
Requires-Dist: typing-extensions
|
|
33
33
|
Provides-Extra: doc
|
|
@@ -75,7 +75,7 @@ $ pip install dartsort
|
|
|
75
75
|
|
|
76
76
|
If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
|
|
77
77
|
|
|
78
|
-
|
|
78
|
+
### Setting up a Python environment
|
|
79
79
|
|
|
80
80
|
If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
|
|
81
81
|
|
|
@@ -32,7 +32,7 @@ $ pip install dartsort
|
|
|
32
32
|
|
|
33
33
|
If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
### Setting up a Python environment
|
|
36
36
|
|
|
37
37
|
If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
|
|
38
38
|
|
|
@@ -32,7 +32,7 @@ $ pip install dartsort
|
|
|
32
32
|
|
|
33
33
|
If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
### Setting up a Python environment
|
|
36
36
|
|
|
37
37
|
If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
|
|
38
38
|
|
|
@@ -56,7 +56,7 @@ class Agglomeration:
|
|
|
56
56
|
def agglomerate(
|
|
57
57
|
*,
|
|
58
58
|
sorting: DARTsortSorting,
|
|
59
|
-
recording: BaseRecording
|
|
59
|
+
recording: BaseRecording,
|
|
60
60
|
template_merge_cfg: TemplateMergeConfig | None,
|
|
61
61
|
refinement_cfg: RefinementConfig | None,
|
|
62
62
|
motion: MotionInfo,
|
|
@@ -104,7 +104,7 @@ def agglomerate(
|
|
|
104
104
|
)
|
|
105
105
|
|
|
106
106
|
# tdist tells us the possible merges
|
|
107
|
-
|
|
107
|
+
distance_mask = linkage_mask(
|
|
108
108
|
tdist.distances,
|
|
109
109
|
linkage_method=template_merge_cfg.linkage,
|
|
110
110
|
threshold=template_merge_cfg.merge_distance_threshold,
|
|
@@ -117,9 +117,9 @@ def agglomerate(
|
|
|
117
117
|
dt=refinement_cfg.glom_firing_corr_dt,
|
|
118
118
|
method=refinement_cfg.glom_firing_corr_method,
|
|
119
119
|
)
|
|
120
|
-
_oldsum =
|
|
120
|
+
_oldsum = distance_mask[np.triu_indices_from(distance_mask)].sum()
|
|
121
121
|
fcorr_mask = fcorr <= refinement_cfg.glom_max_firing_corr
|
|
122
|
-
mask = np.logical_and(
|
|
122
|
+
mask = np.logical_and(distance_mask, fcorr_mask)
|
|
123
123
|
np.fill_diagonal(mask, True)
|
|
124
124
|
_newsum = mask[np.triu_indices_from(mask)].sum()
|
|
125
125
|
logger.dartsortdebug(
|
|
@@ -127,6 +127,7 @@ def agglomerate(
|
|
|
127
127
|
)
|
|
128
128
|
else:
|
|
129
129
|
fcorr = fcorr_mask = None
|
|
130
|
+
mask = distance_mask
|
|
130
131
|
|
|
131
132
|
# restrict mask by overlap criteria
|
|
132
133
|
qda_res = qda(
|
|
@@ -155,23 +156,50 @@ def agglomerate(
|
|
|
155
156
|
axis=0,
|
|
156
157
|
)
|
|
157
158
|
qda_mask = np.logical_or(qda_mask_uni, qda_mask_bi)
|
|
159
|
+
assert np.all(qda_mask <= mask)
|
|
160
|
+
if fcorr_mask is not None:
|
|
161
|
+
assert np.all(np.logical_and(qda_mask, fcorr_mask) <= mask)
|
|
162
|
+
|
|
163
|
+
if refinement_cfg.spikeinterface_merge_preset is not None:
|
|
164
|
+
pair_mask = tdist.distances < refinement_cfg.spikeinterface_merge_max_distance
|
|
165
|
+
if refinement_cfg.spikeinterface_merge_min_coentropy is not None:
|
|
166
|
+
cmask, _ = coentropy_merge_mask(
|
|
167
|
+
sorting=sorting,
|
|
168
|
+
min_coentropy=refinement_cfg.spikeinterface_merge_min_coentropy,
|
|
169
|
+
coverage_threshold=refinement_cfg.spikeinterface_merge_coent_coverage,
|
|
170
|
+
iou_threshold=refinement_cfg.spikeinterface_merge_coent_iou,
|
|
171
|
+
)
|
|
172
|
+
pair_mask = np.logical_or(cmask, pair_mask)
|
|
173
|
+
|
|
174
|
+
si_mask = spikeinterface_merge_mask(
|
|
175
|
+
recording=recording,
|
|
176
|
+
sorting=sorting,
|
|
177
|
+
preset=refinement_cfg.spikeinterface_merge_preset,
|
|
178
|
+
censor_ms=refinement_cfg.censor_ms,
|
|
179
|
+
template_data=tdist.template_data,
|
|
180
|
+
pair_mask=pair_mask,
|
|
181
|
+
)
|
|
182
|
+
else:
|
|
183
|
+
si_mask = None
|
|
184
|
+
|
|
185
|
+
# force merges for very close neighbors
|
|
158
186
|
force_mask = linkage_mask(
|
|
159
187
|
tdist.distances,
|
|
160
188
|
linkage_method=template_merge_cfg.linkage,
|
|
161
189
|
threshold=refinement_cfg.qda_force_merge_for_temp_dist_below,
|
|
162
190
|
)
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
191
|
+
|
|
192
|
+
# extract final mask
|
|
193
|
+
final_mask = np.logical_or(qda_mask, force_mask)
|
|
194
|
+
if si_mask is not None:
|
|
195
|
+
final_mask = np.logical_or(final_mask, si_mask)
|
|
196
|
+
np.fill_diagonal(final_mask, True)
|
|
197
|
+
final_mask_as_distance = np.logical_not(final_mask).astype(np.float32)
|
|
170
198
|
|
|
171
199
|
agg_sorting, new_ids = recluster(
|
|
172
200
|
sorting=sorting,
|
|
173
201
|
unit_ids=tdist.template_data.unit_ids,
|
|
174
|
-
dists=
|
|
202
|
+
dists=final_mask_as_distance,
|
|
175
203
|
shifts=tdist.shifts,
|
|
176
204
|
unit_snrs=tdist.template_data.snrs_by_channel().max(1),
|
|
177
205
|
threshold=0.5,
|
|
@@ -353,6 +381,100 @@ def _get_scores(sorting: DARTsortSorting) -> tuple[np.ndarray, Scores]:
|
|
|
353
381
|
return labels, scores
|
|
354
382
|
|
|
355
383
|
|
|
384
|
+
def spikeinterface_merge_mask(
|
|
385
|
+
*,
|
|
386
|
+
recording: BaseRecording,
|
|
387
|
+
sorting: DARTsortSorting,
|
|
388
|
+
preset: str | None,
|
|
389
|
+
censor_ms: float = 0.0,
|
|
390
|
+
template_data: TemplateData,
|
|
391
|
+
pair_mask: np.ndarray,
|
|
392
|
+
min_count: int = 100,
|
|
393
|
+
):
|
|
394
|
+
from spikeinterface.curation.auto_merge import compute_merge_unit_groups
|
|
395
|
+
from spikeinterface.postprocessing import ComputeTemplateSimilarity
|
|
396
|
+
|
|
397
|
+
# censor first
|
|
398
|
+
if censor_ms:
|
|
399
|
+
sorting = deduplicate_spikes(sorting, censor_ms)
|
|
400
|
+
|
|
401
|
+
# analyzer
|
|
402
|
+
analyzer = sorting.to_sorting_analyzer(
|
|
403
|
+
recording=recording, template_data=template_data
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
# register the mask as the template similarity extension
|
|
407
|
+
tsim_ext = ComputeTemplateSimilarity(analyzer)
|
|
408
|
+
tsim_ext.data = {"similarity": pair_mask.astype(np.float32)}
|
|
409
|
+
tsim_ext.params = {"method": "dartsort"}
|
|
410
|
+
tsim_ext.run_info = {"run_completed": True}
|
|
411
|
+
analyzer.extensions["template_similarity"] = tsim_ext
|
|
412
|
+
|
|
413
|
+
# handle custom presets
|
|
414
|
+
if preset == "dartsort_slay_xc":
|
|
415
|
+
steps = [
|
|
416
|
+
"num_spikes",
|
|
417
|
+
"remove_contaminated",
|
|
418
|
+
"unit_locations",
|
|
419
|
+
"template_similarity",
|
|
420
|
+
"slay_score",
|
|
421
|
+
"cross_contamination",
|
|
422
|
+
"quality_score",
|
|
423
|
+
]
|
|
424
|
+
preset = None
|
|
425
|
+
analyzer.compute_one_extension("correlograms")
|
|
426
|
+
elif preset == "dartsort_slay_ccg":
|
|
427
|
+
steps = [
|
|
428
|
+
"num_spikes",
|
|
429
|
+
"remove_contaminated",
|
|
430
|
+
"unit_locations",
|
|
431
|
+
"template_similarity",
|
|
432
|
+
"correlogram",
|
|
433
|
+
"slay_score",
|
|
434
|
+
"quality_score",
|
|
435
|
+
]
|
|
436
|
+
preset = None
|
|
437
|
+
analyzer.compute_one_extension("correlograms")
|
|
438
|
+
elif preset == "dartsort_slay_xc_ccg":
|
|
439
|
+
steps = [
|
|
440
|
+
"num_spikes",
|
|
441
|
+
"remove_contaminated",
|
|
442
|
+
"unit_locations",
|
|
443
|
+
"template_similarity",
|
|
444
|
+
"correlogram",
|
|
445
|
+
"cross_contamination",
|
|
446
|
+
"slay_score",
|
|
447
|
+
"quality_score",
|
|
448
|
+
]
|
|
449
|
+
preset = None
|
|
450
|
+
analyzer.compute_one_extension("correlograms")
|
|
451
|
+
else:
|
|
452
|
+
assert preset is not None
|
|
453
|
+
steps = None
|
|
454
|
+
|
|
455
|
+
# make parameters aware of censorship and other params
|
|
456
|
+
my_step_params = {
|
|
457
|
+
"num_spikes": {"min_spikes": min_count},
|
|
458
|
+
"remove_contaminated": {"censored_period_ms": censor_ms},
|
|
459
|
+
"template_similarity": {"similarity_method": "dartsort"},
|
|
460
|
+
"correlogram": {"censor_correlograms_ms": censor_ms},
|
|
461
|
+
"cross_contamination": {"censored_period_ms": censor_ms},
|
|
462
|
+
"quality_score": {"censored_period_ms": censor_ms},
|
|
463
|
+
}
|
|
464
|
+
groups = compute_merge_unit_groups(
|
|
465
|
+
preset=preset,
|
|
466
|
+
steps=steps,
|
|
467
|
+
sorting_analyzer=analyzer,
|
|
468
|
+
steps_params=my_step_params,
|
|
469
|
+
force_copy=False,
|
|
470
|
+
)
|
|
471
|
+
mask = np.zeros_like(pair_mask)
|
|
472
|
+
for g in groups:
|
|
473
|
+
g = np.array(g)
|
|
474
|
+
mask[g[:, None], g[None, :]] = True
|
|
475
|
+
return mask
|
|
476
|
+
|
|
477
|
+
|
|
356
478
|
@databag
|
|
357
479
|
class QDAResult:
|
|
358
480
|
"""Unit pair QDA metrics
|
|
@@ -666,7 +788,8 @@ def combine_gmm_scores(
|
|
|
666
788
|
|
|
667
789
|
# check invariants at the top
|
|
668
790
|
if responsibilities.shape[1] > 2:
|
|
669
|
-
|
|
791
|
+
_maxdiff = np.diff(responsibilities[:, :-1], axis=1).max()
|
|
792
|
+
assert _maxdiff <= 1e-3, _maxdiff
|
|
670
793
|
assert np.greater_equal(np.isneginf(logliks[:, :-1]), candidates == -1).all()
|
|
671
794
|
if sorting.labels is not None:
|
|
672
795
|
assert np.all(
|
|
@@ -699,7 +822,8 @@ def combine_gmm_scores(
|
|
|
699
822
|
|
|
700
823
|
# check invariants at the bottom
|
|
701
824
|
if mergedr.shape[1] > 2:
|
|
702
|
-
|
|
825
|
+
_maxdiff = np.diff(mergedr[:, : cand.shape[1]], axis=1).max()
|
|
826
|
+
assert _maxdiff <= 1e-3, _maxdiff
|
|
703
827
|
assert np.greater_equal(np.isneginf(mergedl[:, : cand.shape[1]]), cand == -1).all()
|
|
704
828
|
assert (cand < 0).sum() >= nbye
|
|
705
829
|
if sorting.labels is not None:
|
|
@@ -734,7 +858,7 @@ def _combine_loop(
|
|
|
734
858
|
continue
|
|
735
859
|
|
|
736
860
|
eq_ncandj = rcand[j + 1 :] == ncandj
|
|
737
|
-
if eq_ncandj.sum()
|
|
861
|
+
if eq_ncandj.sum() < 1:
|
|
738
862
|
continue
|
|
739
863
|
|
|
740
864
|
rsum = mergedr[s, j]
|
|
@@ -800,6 +924,8 @@ def deduplicate_spikes(
|
|
|
800
924
|
ndrop = 0
|
|
801
925
|
for unit_id in unit_ids:
|
|
802
926
|
in_unit = np.flatnonzero(new_labels == unit_id)
|
|
927
|
+
if in_unit.size <= 1:
|
|
928
|
+
continue
|
|
803
929
|
t = times_samples[in_unit]
|
|
804
930
|
dt = np.diff(t)
|
|
805
931
|
if dt.min() > radius_samples:
|
|
@@ -870,3 +996,152 @@ def _dedup_unit_loop(
|
|
|
870
996
|
break
|
|
871
997
|
|
|
872
998
|
i0 = i1
|
|
999
|
+
|
|
1000
|
+
|
|
1001
|
+
@databag
|
|
1002
|
+
class CoentropyResult:
|
|
1003
|
+
coentropy: np.ndarray
|
|
1004
|
+
"""KxK; reduction of entropy per cooccurrence due to merging pair"""
|
|
1005
|
+
|
|
1006
|
+
cooccurrence: np.ndarray
|
|
1007
|
+
"""KxK; number of times these units score the same spike"""
|
|
1008
|
+
|
|
1009
|
+
rival_count: np.ndarray
|
|
1010
|
+
"""KxK; number of times one unit scores a spike where the other is top"""
|
|
1011
|
+
|
|
1012
|
+
occurrence: np.ndarray
|
|
1013
|
+
"""K; number of times the unit appears in the candidates at all"""
|
|
1014
|
+
|
|
1015
|
+
cov: np.ndarray
|
|
1016
|
+
"""KxK; rival count / max pair count (rival diag)"""
|
|
1017
|
+
|
|
1018
|
+
iou: np.ndarray
|
|
1019
|
+
"""KxK; rival count over pair sum"""
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
def coentropy_merge_mask(
|
|
1023
|
+
sorting: DARTsortSorting,
|
|
1024
|
+
min_coentropy: float,
|
|
1025
|
+
coverage_threshold: float,
|
|
1026
|
+
iou_threshold: float,
|
|
1027
|
+
gmm_prefix=("merged", "gmm"),
|
|
1028
|
+
) -> tuple[np.ndarray, CoentropyResult]:
|
|
1029
|
+
"""
|
|
1030
|
+
Parameters
|
|
1031
|
+
----------
|
|
1032
|
+
sorting : DARTsortSorting
|
|
1033
|
+
min_coentropy : float
|
|
1034
|
+
Must be met by pair for mask=True
|
|
1035
|
+
min_coverage : float
|
|
1036
|
+
Pairs such that at least one unit in each pair has
|
|
1037
|
+
rival_count/count > mincov are allowed
|
|
1038
|
+
iou_threshold: float
|
|
1039
|
+
Pairs with rival iou > iouthresh are allowed
|
|
1040
|
+
"""
|
|
1041
|
+
c = coentropy(sorting, gmm_prefix=gmm_prefix)
|
|
1042
|
+
assert c is not None
|
|
1043
|
+
|
|
1044
|
+
mask = np.logical_or(c.cov >= coverage_threshold, c.iou >= iou_threshold)
|
|
1045
|
+
mask = np.logical_and(c.coentropy >= min_coentropy, mask)
|
|
1046
|
+
np.fill_diagonal(mask, True)
|
|
1047
|
+
return mask, c
|
|
1048
|
+
|
|
1049
|
+
|
|
1050
|
+
def coentropy(
|
|
1051
|
+
sorting: DARTsortSorting,
|
|
1052
|
+
gmm_prefix=("merged", "gmm"),
|
|
1053
|
+
) -> CoentropyResult | None:
|
|
1054
|
+
"""Calculate entropy reduction due to merging pairs."""
|
|
1055
|
+
for k in gmm_prefix:
|
|
1056
|
+
cands = getattr(sorting, f"{k}_candidates", None)
|
|
1057
|
+
resps = getattr(sorting, f"{k}_responsibilities", None)
|
|
1058
|
+
if cands is not None:
|
|
1059
|
+
assert resps is not None
|
|
1060
|
+
break
|
|
1061
|
+
else:
|
|
1062
|
+
return None
|
|
1063
|
+
|
|
1064
|
+
k = sorting.n_units
|
|
1065
|
+
resps = resps[:, : cands.shape[1]].astype(np.float64)
|
|
1066
|
+
coentropy = np.zeros((k, k))
|
|
1067
|
+
cooccurrence = np.zeros((k, k), dtype=np.int64)
|
|
1068
|
+
rival_count = np.zeros((k, k), dtype=np.int64)
|
|
1069
|
+
occurrence = np.zeros((k,), dtype=np.int64)
|
|
1070
|
+
_calc_coentropy(coentropy, cooccurrence, rival_count, occurrence, cands, resps)
|
|
1071
|
+
rival_count += rival_count.T
|
|
1072
|
+
cdiag = np.diagonal(rival_count)
|
|
1073
|
+
assert (cdiag % 2 == 0).all()
|
|
1074
|
+
np.fill_diagonal(rival_count, cdiag // 2)
|
|
1075
|
+
coentropy += coentropy.T
|
|
1076
|
+
cooccurrence += cooccurrence.T
|
|
1077
|
+
|
|
1078
|
+
# rival count diagonal is just unit top count (not exactly label count,
|
|
1079
|
+
# since it doesn't account for noise assignments)
|
|
1080
|
+
counts = np.diagonal(rival_count)
|
|
1081
|
+
counts = np.maximum(counts, 1)
|
|
1082
|
+
|
|
1083
|
+
cov = rival_count / counts
|
|
1084
|
+
cov = np.minimum(cov, cov.T)
|
|
1085
|
+
|
|
1086
|
+
# this is a disjoint union, since it's the top-label count
|
|
1087
|
+
union = counts[:, None] + counts[None, :]
|
|
1088
|
+
iou = rival_count / union
|
|
1089
|
+
|
|
1090
|
+
return CoentropyResult(
|
|
1091
|
+
coentropy=coentropy,
|
|
1092
|
+
cooccurrence=cooccurrence,
|
|
1093
|
+
rival_count=rival_count,
|
|
1094
|
+
occurrence=occurrence,
|
|
1095
|
+
cov=cov,
|
|
1096
|
+
iou=iou,
|
|
1097
|
+
)
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
@numba.njit(parallel=True)
|
|
1101
|
+
def _calc_coentropy(
|
|
1102
|
+
coentropy: np.ndarray,
|
|
1103
|
+
cooccurrence: np.ndarray,
|
|
1104
|
+
rival_count: np.ndarray,
|
|
1105
|
+
occurrence: np.ndarray,
|
|
1106
|
+
cands: np.ndarray,
|
|
1107
|
+
resps: np.ndarray,
|
|
1108
|
+
):
|
|
1109
|
+
for i in numba.prange(cands.shape[0]): # ty: ignore
|
|
1110
|
+
u = cands[i]
|
|
1111
|
+
q = resps[i]
|
|
1112
|
+
log_q = np.log(q)
|
|
1113
|
+
np.nan_to_num(log_q, copy=False, neginf=0.0)
|
|
1114
|
+
dh = q * log_q
|
|
1115
|
+
|
|
1116
|
+
ui0 = u[0]
|
|
1117
|
+
qi0 = q[0]
|
|
1118
|
+
dhi0 = dh[0]
|
|
1119
|
+
|
|
1120
|
+
occurrence[ui0] += 1
|
|
1121
|
+
rival_count[ui0, ui0] += 1
|
|
1122
|
+
|
|
1123
|
+
for j in range(1, cands.shape[1]):
|
|
1124
|
+
uj = u[j]
|
|
1125
|
+
if uj < 0:
|
|
1126
|
+
break
|
|
1127
|
+
|
|
1128
|
+
ii = min(ui0, uj)
|
|
1129
|
+
jj = max(ui0, uj)
|
|
1130
|
+
|
|
1131
|
+
occurrence[uj] += 1
|
|
1132
|
+
rival_count[ui0, uj] += 1
|
|
1133
|
+
|
|
1134
|
+
cij = cooccurrence[ii, jj] + 1
|
|
1135
|
+
cooccurrence[ii, jj] = cij
|
|
1136
|
+
|
|
1137
|
+
# change in entropy due to merging uj, uk:
|
|
1138
|
+
# subtract their current contribution, add the new contribution
|
|
1139
|
+
# we want reduction of entropy, so this is the negative of that!
|
|
1140
|
+
qij = q[j] + qi0
|
|
1141
|
+
dhij = dh[j] + dhi0
|
|
1142
|
+
if qij > 0:
|
|
1143
|
+
dhij -= qij * np.log(qij)
|
|
1144
|
+
|
|
1145
|
+
# Welford mean of -dh
|
|
1146
|
+
cur_coent = coentropy[ii, jj]
|
|
1147
|
+
coentropy[ii, jj] = cur_coent + (-dhij - cur_coent) / cij
|
|
@@ -19,7 +19,16 @@ from ..util.job_util import ensure_computation_config
|
|
|
19
19
|
from ..util.main_util import ds_save_intermediate_labels
|
|
20
20
|
from ..util.motion import MotionInfo
|
|
21
21
|
from ..util.multiprocessing_util import handle_negative_jobs
|
|
22
|
-
from . import
|
|
22
|
+
from ..util.torch_util import cleanup_and_log_gpu_usage
|
|
23
|
+
from . import (
|
|
24
|
+
agglomerate,
|
|
25
|
+
cluster_util,
|
|
26
|
+
density,
|
|
27
|
+
forward_backward,
|
|
28
|
+
kmeans,
|
|
29
|
+
mixture,
|
|
30
|
+
refine_util,
|
|
31
|
+
)
|
|
23
32
|
from .clustering_features import SimpleMatrixFeatures, StableWaveformFeatures
|
|
24
33
|
|
|
25
34
|
if TYPE_CHECKING:
|
|
@@ -292,6 +301,10 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
292
301
|
workers=-1,
|
|
293
302
|
uhdversion=False,
|
|
294
303
|
random_seed=0,
|
|
304
|
+
kmeans_cleanup=True,
|
|
305
|
+
kmeans_max_sigma: float = 5.0,
|
|
306
|
+
kmeans_iter=100,
|
|
307
|
+
device=None,
|
|
295
308
|
**kwargs,
|
|
296
309
|
):
|
|
297
310
|
super().__init__(**kwargs)
|
|
@@ -308,6 +321,10 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
308
321
|
self.workers = workers
|
|
309
322
|
self.uhdversion = uhdversion
|
|
310
323
|
self.random_seed = random_seed
|
|
324
|
+
self.kmeans_cleanup = kmeans_cleanup
|
|
325
|
+
self.kmeans_max_sigma = kmeans_max_sigma
|
|
326
|
+
self.kmeans_iter = kmeans_iter
|
|
327
|
+
self.device = device
|
|
311
328
|
|
|
312
329
|
@classmethod
|
|
313
330
|
def from_config(
|
|
@@ -335,6 +352,7 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
335
352
|
outlier_radius=clustering_cfg.outlier_radius,
|
|
336
353
|
outlier_neighbor_count=clustering_cfg.outlier_neighbor_count,
|
|
337
354
|
workers=workers,
|
|
355
|
+
device=computation_cfg.actual_device(),
|
|
338
356
|
uhdversion=uhdversion,
|
|
339
357
|
computation_cfg=computation_cfg,
|
|
340
358
|
waveform_cfg=waveform_cfg,
|
|
@@ -342,6 +360,9 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
342
360
|
save_labels_dir=save_labels_dir,
|
|
343
361
|
labels_fmt=labels_fmt,
|
|
344
362
|
sampling_cfg=clustering_cfg.sampling_cfg,
|
|
363
|
+
kmeans_cleanup=clustering_cfg.dpc_kmeans_cleanup,
|
|
364
|
+
kmeans_iter=clustering_cfg.kmeans_iter,
|
|
365
|
+
kmeans_max_sigma=clustering_cfg.gmmdpc_max_sigma,
|
|
345
366
|
)
|
|
346
367
|
|
|
347
368
|
def _cluster(
|
|
@@ -425,6 +446,19 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
425
446
|
kdtree = None
|
|
426
447
|
labels = res["labels"]
|
|
427
448
|
|
|
449
|
+
if self.kmeans_cleanup:
|
|
450
|
+
kres = kmeans.truncated_kmeans_from_labels(
|
|
451
|
+
X=X,
|
|
452
|
+
labels=labels,
|
|
453
|
+
device=self.device,
|
|
454
|
+
max_sigma=self.kmeans_max_sigma,
|
|
455
|
+
n_iter=self.kmeans_iter,
|
|
456
|
+
)
|
|
457
|
+
assert kres.labels is not None
|
|
458
|
+
labels = kres.labels.numpy(force=True)
|
|
459
|
+
del kres
|
|
460
|
+
cleanup_and_log_gpu_usage(self.computation_cfg, "DPC->kmeans")
|
|
461
|
+
|
|
428
462
|
labels = cluster_util.decrumb(
|
|
429
463
|
labels, min_size=self.remove_clusters_smaller_than, in_place=True
|
|
430
464
|
)
|
|
@@ -821,6 +855,7 @@ class AgglomerateRefinement(Refinement):
|
|
|
821
855
|
recording: BaseRecording | None,
|
|
822
856
|
motion: MotionInfo,
|
|
823
857
|
):
|
|
858
|
+
assert recording is not None
|
|
824
859
|
return agglomerate.agglomerate(
|
|
825
860
|
recording=recording,
|
|
826
861
|
sorting=sorting,
|
|
@@ -25,15 +25,6 @@ from ..util.py_util import databag
|
|
|
25
25
|
from ..util.waveform_util import single_channel_index
|
|
26
26
|
from . import cluster_util
|
|
27
27
|
|
|
28
|
-
default_clustering_features_cfg = ClusteringFeaturesConfig()
|
|
29
|
-
minimal_features_cfg = ClusteringFeaturesConfig(
|
|
30
|
-
n_main_channel_pcs=0,
|
|
31
|
-
use_amplitude=False,
|
|
32
|
-
use_signed_amplitude=False,
|
|
33
|
-
use_x=False,
|
|
34
|
-
use_z=False,
|
|
35
|
-
)
|
|
36
|
-
|
|
37
28
|
logger = get_logger(__name__)
|
|
38
29
|
|
|
39
30
|
|
|
@@ -77,8 +68,14 @@ class SimpleMatrixFeatures:
|
|
|
77
68
|
)
|
|
78
69
|
if xyza is not None:
|
|
79
70
|
x = xyza[:, 0]
|
|
71
|
+
if not _allfinite(x):
|
|
72
|
+
raise ValueError(_numbers_error_str("x", x))
|
|
80
73
|
z = xyza[:, 2]
|
|
74
|
+
if not _allfinite(z):
|
|
75
|
+
raise ValueError(_numbers_error_str("z", z))
|
|
81
76
|
z_reg = motion.correct_s(t_s, z)
|
|
77
|
+
if not _allfinite(z_reg):
|
|
78
|
+
raise ValueError(_numbers_error_str("z_reg", z_reg))
|
|
82
79
|
else:
|
|
83
80
|
x = z = z_reg = None
|
|
84
81
|
|
|
@@ -99,10 +96,14 @@ class SimpleMatrixFeatures:
|
|
|
99
96
|
amp = getattr(sorting, clustering_features_cfg.amplitudes_dataset_name)
|
|
100
97
|
if clustering_features_cfg.use_amplitude:
|
|
101
98
|
assert amp is not None
|
|
99
|
+
if not _allfinite(amp):
|
|
100
|
+
raise ValueError(_numbers_error_str("amp", amp))
|
|
102
101
|
ampft = amp.copy()
|
|
103
102
|
if clustering_features_cfg.log_transform_amplitude:
|
|
104
103
|
ampft = np.log(clustering_features_cfg.amp_log_c + ampft)
|
|
105
104
|
ampft *= clustering_features_cfg.amp_scale
|
|
105
|
+
if not _allfinite(ampft):
|
|
106
|
+
raise ValueError(_numbers_error_str("ampft", ampft))
|
|
106
107
|
features.append(ampft[:, None])
|
|
107
108
|
|
|
108
109
|
v = getattr(sorting, clustering_features_cfg.voltages_dataset_name, None)
|
|
@@ -123,6 +124,8 @@ class SimpleMatrixFeatures:
|
|
|
123
124
|
rank=clustering_features_cfg.n_main_channel_pcs,
|
|
124
125
|
dataset_name=clustering_features_cfg.pca_dataset_name,
|
|
125
126
|
)
|
|
127
|
+
if not _allfinite(pcs):
|
|
128
|
+
raise ValueError(_numbers_error_str("No motion pcs", pcs))
|
|
126
129
|
elif do_pcs and clustering_features_cfg.motion_aware:
|
|
127
130
|
shifts, n_pitches_shift = motion.pitch_shifts(
|
|
128
131
|
sorting=sorting,
|
|
@@ -141,6 +144,8 @@ class SimpleMatrixFeatures:
|
|
|
141
144
|
mask = np.broadcast_to(mask, len(schan))
|
|
142
145
|
if hasattr(sorting, clustering_features_cfg.pca_dataset_name):
|
|
143
146
|
pcs = getattr(sorting, clustering_features_cfg.pca_dataset_name)
|
|
147
|
+
if not _allfinite(pcs):
|
|
148
|
+
raise ValueError(_numbers_error_str("sorting pcs", pcs))
|
|
144
149
|
erp, pcs = interpolate_by_chunk(
|
|
145
150
|
mask=mask,
|
|
146
151
|
dataset=pcs,
|
|
@@ -153,6 +158,8 @@ class SimpleMatrixFeatures:
|
|
|
153
158
|
params=clustering_features_cfg.interp_params,
|
|
154
159
|
)
|
|
155
160
|
pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
|
|
161
|
+
if not _allfinite(pcs):
|
|
162
|
+
raise ValueError(_numbers_error_str("sorting interp pcs", pcs))
|
|
156
163
|
else:
|
|
157
164
|
assert sorting.parent_h5_path is not None
|
|
158
165
|
with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
|
|
@@ -168,20 +175,25 @@ class SimpleMatrixFeatures:
|
|
|
168
175
|
params=clustering_features_cfg.interp_params,
|
|
169
176
|
)
|
|
170
177
|
pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
|
|
178
|
+
if not _allfinite(pcs):
|
|
179
|
+
raise ValueError(_numbers_error_str("h5 interp pcs", pcs))
|
|
171
180
|
|
|
172
181
|
if do_pcs:
|
|
173
182
|
assert pcs is not None
|
|
174
|
-
|
|
183
|
+
pctf = clustering_features_cfg.pc_transform
|
|
184
|
+
if pctf == "log":
|
|
175
185
|
pcs = signed_log1p(
|
|
176
186
|
pcs, pre_scale=clustering_features_cfg.pc_pre_transform_scale
|
|
177
187
|
)
|
|
178
|
-
elif
|
|
188
|
+
elif pctf == "sqrt":
|
|
179
189
|
pcs = signed_sqrt_transform(
|
|
180
190
|
pcs, pre_scale=clustering_features_cfg.pc_pre_transform_scale
|
|
181
191
|
)
|
|
182
192
|
else:
|
|
183
|
-
assert
|
|
193
|
+
assert pctf in ("none", None)
|
|
184
194
|
pcs *= clustering_features_cfg.pc_scale
|
|
195
|
+
if not _allfinite(pcs):
|
|
196
|
+
raise ValueError(_numbers_error_str(f"{pctf} pcs", pcs))
|
|
185
197
|
if torch.is_tensor(pcs):
|
|
186
198
|
pcs = pcs.numpy(force=True)
|
|
187
199
|
features.append(pcs)
|
|
@@ -310,3 +322,16 @@ def signed_sqrt_transform(x, pre_scale=1.0):
|
|
|
310
322
|
xx.sub_(1.0)
|
|
311
323
|
xx.mul_(torch.sign(x))
|
|
312
324
|
return xx
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _allfinite(x):
|
|
328
|
+
if isinstance(x, torch.Tensor):
|
|
329
|
+
return x.isfinite().all()
|
|
330
|
+
else:
|
|
331
|
+
return np.isfinite(x).all()
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _numbers_error_str(name: str, x: np.ndarray):
|
|
335
|
+
if isinstance(x, torch.Tensor):
|
|
336
|
+
x = x.numpy(force=True)
|
|
337
|
+
return f"{name}: {np.isposinf(x).sum()} +inf, {np.isneginf(x).sum()} -inf, {np.isnan(x).sum()} nan."
|