DARTsort 0.5.9__tar.gz → 0.5.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.9 → dartsort-0.5.10}/PKG-INFO +2 -2
- {dartsort-0.5.9 → dartsort-0.5.10}/README.md +1 -1
- {dartsort-0.5.9 → dartsort-0.5.10}/docs/index.md +1 -1
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/agglomerate.py +287 -11
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/clustering.py +36 -1
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/clustering_features.py +37 -12
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/kmeans.py +296 -5
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/mixture.py +192 -58
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/config.py +13 -6
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/evaluate/analysis.py +19 -7
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/evaluate/comparison.py +33 -4
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/main.py +17 -10
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/grab.py +89 -16
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/matching.py +24 -6
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/matching_util/compressed_upsampled.py +12 -1
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/matching_util/drifty.py +79 -37
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/matching_util/matching_base.py +7 -2
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/peel_base.py +55 -16
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/peel_lib.py +63 -29
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/reduction_template.py +14 -9
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/subtract.py +51 -10
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/templates/get_templates.py +2 -2
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/templates/postprocess_util.py +4 -4
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/templates/template_util.py +7 -3
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/templates/templates.py +10 -4
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/templates/templib.py +5 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/_multichan_denoiser_kit.py +12 -2
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/decollider.py +62 -12
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/pipeline.py +7 -1
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/temporal_pca.py +39 -14
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/transform_base.py +22 -3
- dartsort-0.5.10/src/dartsort/transform/whiten.py +102 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/data_util.py +171 -1
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/internal_config.py +81 -31
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/main_util.py +1 -1
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/noise_util.py +180 -27
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/spiketorch.py +24 -5
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/testing_util/matching_debug_util.py +40 -6
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/analysis_plots.py +166 -5
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/gt.py +0 -3
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/mixture.py +6 -6
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/unit.py +174 -71
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/unit_comparison.py +12 -6
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort.egg-info/PKG-INFO +2 -2
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_config.py +2 -1
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_kmeans.py +7 -4
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_matching.py +2 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_subtract.py +8 -9
- dartsort-0.5.9/src/dartsort/transform/whiten.py +0 -34
- {dartsort-0.5.9 → dartsort-0.5.10}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/.gitignore +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/docs/main_api.md +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/environment.yml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/.keep +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/basic_integration_test.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/ci diag.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/dartsort_dredge_demo.ipynb +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/dartsort_dredge_demo.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/data test.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/dead reckoning.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/denoise np1.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/denoise np2.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/exploratory.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/geom debug.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/hdbscan_clustering_5min.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/hybrid 10.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/hybrid cluster.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/hybrid_11.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/initial training.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/make hybrid.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/mgh.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/np1 feats.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/np2 processing version c.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/paperfigs.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/pdists.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/postcheck.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/ptp vae.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/re clust.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/reg fig loop.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/reg_sim_exp.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/relocating.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/relocating_all.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/restats +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/sortStinger1.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/sortStinger3.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/sortStinger4.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/sortStinger5.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/sortStinger6.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/spike farm.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/ss_ensemble_sim.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/ss_ensemble_sim_drift.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/ss_sim.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/stan diagnostics.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/storytelling c.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/storytelling.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/tests/debug_matching.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/tests/test_agreement.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/tests/test_amplitude_scaling.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/tests/test_batch_optim.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/tests/test_ibme.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/tests/test_mm_initialization.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/tests/test_normxcorr1d.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/tests/test_reassignment.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/tests/test_relocation.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/notebook/thomas.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/pyproject.toml +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/setup.cfg +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/cluster_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/modes.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/clustering/refine_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/evaluate/simkit.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/mixture_classifier.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/motion.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/vismain.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort.egg-info/SOURCES.txt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/conftest.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_alignment.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_clustering.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_data_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_density.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_detect.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_linalg.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_modeling.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_simkit.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_templates.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_threshold.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_transform.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.9 → dartsort-0.5.10}/zensical.toml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dartsort
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.10
|
|
4
4
|
Summary: DARTsort
|
|
5
5
|
Author-email: Charlie Windolf <ciw2048@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/cwindolf/dartsort
|
|
@@ -75,7 +75,7 @@ $ pip install dartsort
|
|
|
75
75
|
|
|
76
76
|
If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
|
|
77
77
|
|
|
78
|
-
|
|
78
|
+
### Setting up a Python environment
|
|
79
79
|
|
|
80
80
|
If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
|
|
81
81
|
|
|
@@ -32,7 +32,7 @@ $ pip install dartsort
|
|
|
32
32
|
|
|
33
33
|
If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
### Setting up a Python environment
|
|
36
36
|
|
|
37
37
|
If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
|
|
38
38
|
|
|
@@ -32,7 +32,7 @@ $ pip install dartsort
|
|
|
32
32
|
|
|
33
33
|
If you want to run the test suite or use `dartsort.vis`, you can install the optional dependencies with `pip install dartsort[test,vis]`.
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
### Setting up a Python environment
|
|
36
36
|
|
|
37
37
|
If you need to set up Python or PyTorch, I find that a [`conda-forge`](https://conda-forge.org/)-based distribution is the most reliable at installing the GPU dependencies which PyTorch needs (note: `conda-forge` is different from the non-free Anaconda).
|
|
38
38
|
|
|
@@ -56,7 +56,7 @@ class Agglomeration:
|
|
|
56
56
|
def agglomerate(
|
|
57
57
|
*,
|
|
58
58
|
sorting: DARTsortSorting,
|
|
59
|
-
recording: BaseRecording
|
|
59
|
+
recording: BaseRecording,
|
|
60
60
|
template_merge_cfg: TemplateMergeConfig | None,
|
|
61
61
|
refinement_cfg: RefinementConfig | None,
|
|
62
62
|
motion: MotionInfo,
|
|
@@ -104,7 +104,7 @@ def agglomerate(
|
|
|
104
104
|
)
|
|
105
105
|
|
|
106
106
|
# tdist tells us the possible merges
|
|
107
|
-
|
|
107
|
+
distance_mask = linkage_mask(
|
|
108
108
|
tdist.distances,
|
|
109
109
|
linkage_method=template_merge_cfg.linkage,
|
|
110
110
|
threshold=template_merge_cfg.merge_distance_threshold,
|
|
@@ -117,9 +117,9 @@ def agglomerate(
|
|
|
117
117
|
dt=refinement_cfg.glom_firing_corr_dt,
|
|
118
118
|
method=refinement_cfg.glom_firing_corr_method,
|
|
119
119
|
)
|
|
120
|
-
_oldsum =
|
|
120
|
+
_oldsum = distance_mask[np.triu_indices_from(distance_mask)].sum()
|
|
121
121
|
fcorr_mask = fcorr <= refinement_cfg.glom_max_firing_corr
|
|
122
|
-
mask = np.logical_and(
|
|
122
|
+
mask = np.logical_and(distance_mask, fcorr_mask)
|
|
123
123
|
np.fill_diagonal(mask, True)
|
|
124
124
|
_newsum = mask[np.triu_indices_from(mask)].sum()
|
|
125
125
|
logger.dartsortdebug(
|
|
@@ -127,6 +127,7 @@ def agglomerate(
|
|
|
127
127
|
)
|
|
128
128
|
else:
|
|
129
129
|
fcorr = fcorr_mask = None
|
|
130
|
+
mask = distance_mask
|
|
130
131
|
|
|
131
132
|
# restrict mask by overlap criteria
|
|
132
133
|
qda_res = qda(
|
|
@@ -158,19 +159,47 @@ def agglomerate(
|
|
|
158
159
|
assert np.all(qda_mask <= mask)
|
|
159
160
|
if fcorr_mask is not None:
|
|
160
161
|
assert np.all(np.logical_and(qda_mask, fcorr_mask) <= mask)
|
|
162
|
+
|
|
163
|
+
if refinement_cfg.spikeinterface_merge_preset is not None:
|
|
164
|
+
pair_mask = tdist.distances < refinement_cfg.spikeinterface_merge_max_distance
|
|
165
|
+
if refinement_cfg.spikeinterface_merge_min_coentropy is not None:
|
|
166
|
+
cmask, _ = coentropy_merge_mask(
|
|
167
|
+
sorting=sorting,
|
|
168
|
+
min_coentropy=refinement_cfg.spikeinterface_merge_min_coentropy,
|
|
169
|
+
coverage_threshold=refinement_cfg.spikeinterface_merge_coent_coverage,
|
|
170
|
+
iou_threshold=refinement_cfg.spikeinterface_merge_coent_iou,
|
|
171
|
+
)
|
|
172
|
+
pair_mask = np.logical_or(cmask, pair_mask)
|
|
173
|
+
|
|
174
|
+
si_mask = spikeinterface_merge_mask(
|
|
175
|
+
recording=recording,
|
|
176
|
+
sorting=sorting,
|
|
177
|
+
preset=refinement_cfg.spikeinterface_merge_preset,
|
|
178
|
+
censor_ms=refinement_cfg.censor_ms,
|
|
179
|
+
template_data=tdist.template_data,
|
|
180
|
+
pair_mask=pair_mask,
|
|
181
|
+
)
|
|
182
|
+
else:
|
|
183
|
+
si_mask = None
|
|
184
|
+
|
|
185
|
+
# force merges for very close neighbors
|
|
161
186
|
force_mask = linkage_mask(
|
|
162
187
|
tdist.distances,
|
|
163
188
|
linkage_method=template_merge_cfg.linkage,
|
|
164
189
|
threshold=refinement_cfg.qda_force_merge_for_temp_dist_below,
|
|
165
190
|
)
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
191
|
+
|
|
192
|
+
# extract final mask
|
|
193
|
+
final_mask = np.logical_or(qda_mask, force_mask)
|
|
194
|
+
if si_mask is not None:
|
|
195
|
+
final_mask = np.logical_or(final_mask, si_mask)
|
|
196
|
+
np.fill_diagonal(final_mask, True)
|
|
197
|
+
final_mask_as_distance = np.logical_not(final_mask).astype(np.float32)
|
|
169
198
|
|
|
170
199
|
agg_sorting, new_ids = recluster(
|
|
171
200
|
sorting=sorting,
|
|
172
201
|
unit_ids=tdist.template_data.unit_ids,
|
|
173
|
-
dists=
|
|
202
|
+
dists=final_mask_as_distance,
|
|
174
203
|
shifts=tdist.shifts,
|
|
175
204
|
unit_snrs=tdist.template_data.snrs_by_channel().max(1),
|
|
176
205
|
threshold=0.5,
|
|
@@ -352,6 +381,100 @@ def _get_scores(sorting: DARTsortSorting) -> tuple[np.ndarray, Scores]:
|
|
|
352
381
|
return labels, scores
|
|
353
382
|
|
|
354
383
|
|
|
384
|
+
def spikeinterface_merge_mask(
|
|
385
|
+
*,
|
|
386
|
+
recording: BaseRecording,
|
|
387
|
+
sorting: DARTsortSorting,
|
|
388
|
+
preset: str | None,
|
|
389
|
+
censor_ms: float = 0.0,
|
|
390
|
+
template_data: TemplateData,
|
|
391
|
+
pair_mask: np.ndarray,
|
|
392
|
+
min_count: int = 100,
|
|
393
|
+
):
|
|
394
|
+
from spikeinterface.curation.auto_merge import compute_merge_unit_groups
|
|
395
|
+
from spikeinterface.postprocessing import ComputeTemplateSimilarity
|
|
396
|
+
|
|
397
|
+
# censor first
|
|
398
|
+
if censor_ms:
|
|
399
|
+
sorting = deduplicate_spikes(sorting, censor_ms)
|
|
400
|
+
|
|
401
|
+
# analyzer
|
|
402
|
+
analyzer = sorting.to_sorting_analyzer(
|
|
403
|
+
recording=recording, template_data=template_data
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
# register the mask as the template similarity extension
|
|
407
|
+
tsim_ext = ComputeTemplateSimilarity(analyzer)
|
|
408
|
+
tsim_ext.data = {"similarity": pair_mask.astype(np.float32)}
|
|
409
|
+
tsim_ext.params = {"method": "dartsort"}
|
|
410
|
+
tsim_ext.run_info = {"run_completed": True}
|
|
411
|
+
analyzer.extensions["template_similarity"] = tsim_ext
|
|
412
|
+
|
|
413
|
+
# handle custom presets
|
|
414
|
+
if preset == "dartsort_slay_xc":
|
|
415
|
+
steps = [
|
|
416
|
+
"num_spikes",
|
|
417
|
+
"remove_contaminated",
|
|
418
|
+
"unit_locations",
|
|
419
|
+
"template_similarity",
|
|
420
|
+
"slay_score",
|
|
421
|
+
"cross_contamination",
|
|
422
|
+
"quality_score",
|
|
423
|
+
]
|
|
424
|
+
preset = None
|
|
425
|
+
analyzer.compute_one_extension("correlograms")
|
|
426
|
+
elif preset == "dartsort_slay_ccg":
|
|
427
|
+
steps = [
|
|
428
|
+
"num_spikes",
|
|
429
|
+
"remove_contaminated",
|
|
430
|
+
"unit_locations",
|
|
431
|
+
"template_similarity",
|
|
432
|
+
"correlogram",
|
|
433
|
+
"slay_score",
|
|
434
|
+
"quality_score",
|
|
435
|
+
]
|
|
436
|
+
preset = None
|
|
437
|
+
analyzer.compute_one_extension("correlograms")
|
|
438
|
+
elif preset == "dartsort_slay_xc_ccg":
|
|
439
|
+
steps = [
|
|
440
|
+
"num_spikes",
|
|
441
|
+
"remove_contaminated",
|
|
442
|
+
"unit_locations",
|
|
443
|
+
"template_similarity",
|
|
444
|
+
"correlogram",
|
|
445
|
+
"cross_contamination",
|
|
446
|
+
"slay_score",
|
|
447
|
+
"quality_score",
|
|
448
|
+
]
|
|
449
|
+
preset = None
|
|
450
|
+
analyzer.compute_one_extension("correlograms")
|
|
451
|
+
else:
|
|
452
|
+
assert preset is not None
|
|
453
|
+
steps = None
|
|
454
|
+
|
|
455
|
+
# make parameters aware of censorship and other params
|
|
456
|
+
my_step_params = {
|
|
457
|
+
"num_spikes": {"min_spikes": min_count},
|
|
458
|
+
"remove_contaminated": {"censored_period_ms": censor_ms},
|
|
459
|
+
"template_similarity": {"similarity_method": "dartsort"},
|
|
460
|
+
"correlogram": {"censor_correlograms_ms": censor_ms},
|
|
461
|
+
"cross_contamination": {"censored_period_ms": censor_ms},
|
|
462
|
+
"quality_score": {"censored_period_ms": censor_ms},
|
|
463
|
+
}
|
|
464
|
+
groups = compute_merge_unit_groups(
|
|
465
|
+
preset=preset,
|
|
466
|
+
steps=steps,
|
|
467
|
+
sorting_analyzer=analyzer,
|
|
468
|
+
steps_params=my_step_params,
|
|
469
|
+
force_copy=False,
|
|
470
|
+
)
|
|
471
|
+
mask = np.zeros_like(pair_mask)
|
|
472
|
+
for g in groups:
|
|
473
|
+
g = np.array(g)
|
|
474
|
+
mask[g[:, None], g[None, :]] = True
|
|
475
|
+
return mask
|
|
476
|
+
|
|
477
|
+
|
|
355
478
|
@databag
|
|
356
479
|
class QDAResult:
|
|
357
480
|
"""Unit pair QDA metrics
|
|
@@ -665,7 +788,8 @@ def combine_gmm_scores(
|
|
|
665
788
|
|
|
666
789
|
# check invariants at the top
|
|
667
790
|
if responsibilities.shape[1] > 2:
|
|
668
|
-
|
|
791
|
+
_maxdiff = np.diff(responsibilities[:, :-1], axis=1).max()
|
|
792
|
+
assert _maxdiff <= 1e-3, _maxdiff
|
|
669
793
|
assert np.greater_equal(np.isneginf(logliks[:, :-1]), candidates == -1).all()
|
|
670
794
|
if sorting.labels is not None:
|
|
671
795
|
assert np.all(
|
|
@@ -698,7 +822,8 @@ def combine_gmm_scores(
|
|
|
698
822
|
|
|
699
823
|
# check invariants at the bottom
|
|
700
824
|
if mergedr.shape[1] > 2:
|
|
701
|
-
|
|
825
|
+
_maxdiff = np.diff(mergedr[:, : cand.shape[1]], axis=1).max()
|
|
826
|
+
assert _maxdiff <= 1e-3, _maxdiff
|
|
702
827
|
assert np.greater_equal(np.isneginf(mergedl[:, : cand.shape[1]]), cand == -1).all()
|
|
703
828
|
assert (cand < 0).sum() >= nbye
|
|
704
829
|
if sorting.labels is not None:
|
|
@@ -733,7 +858,7 @@ def _combine_loop(
|
|
|
733
858
|
continue
|
|
734
859
|
|
|
735
860
|
eq_ncandj = rcand[j + 1 :] == ncandj
|
|
736
|
-
if eq_ncandj.sum()
|
|
861
|
+
if eq_ncandj.sum() < 1:
|
|
737
862
|
continue
|
|
738
863
|
|
|
739
864
|
rsum = mergedr[s, j]
|
|
@@ -799,6 +924,8 @@ def deduplicate_spikes(
|
|
|
799
924
|
ndrop = 0
|
|
800
925
|
for unit_id in unit_ids:
|
|
801
926
|
in_unit = np.flatnonzero(new_labels == unit_id)
|
|
927
|
+
if in_unit.size <= 1:
|
|
928
|
+
continue
|
|
802
929
|
t = times_samples[in_unit]
|
|
803
930
|
dt = np.diff(t)
|
|
804
931
|
if dt.min() > radius_samples:
|
|
@@ -869,3 +996,152 @@ def _dedup_unit_loop(
|
|
|
869
996
|
break
|
|
870
997
|
|
|
871
998
|
i0 = i1
|
|
999
|
+
|
|
1000
|
+
|
|
1001
|
+
@databag
|
|
1002
|
+
class CoentropyResult:
|
|
1003
|
+
coentropy: np.ndarray
|
|
1004
|
+
"""KxK; reduction of entropy per cooccurrence due to merging pair"""
|
|
1005
|
+
|
|
1006
|
+
cooccurrence: np.ndarray
|
|
1007
|
+
"""KxK; number of times these units score the same spike"""
|
|
1008
|
+
|
|
1009
|
+
rival_count: np.ndarray
|
|
1010
|
+
"""KxK; number of times one unit scores a spike where the other is top"""
|
|
1011
|
+
|
|
1012
|
+
occurrence: np.ndarray
|
|
1013
|
+
"""K; number of times the unit appears in the candidates at all"""
|
|
1014
|
+
|
|
1015
|
+
cov: np.ndarray
|
|
1016
|
+
"""KxK; rival count / max pair count (rival diag)"""
|
|
1017
|
+
|
|
1018
|
+
iou: np.ndarray
|
|
1019
|
+
"""KxK; rival count over pair sum"""
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
def coentropy_merge_mask(
|
|
1023
|
+
sorting: DARTsortSorting,
|
|
1024
|
+
min_coentropy: float,
|
|
1025
|
+
coverage_threshold: float,
|
|
1026
|
+
iou_threshold: float,
|
|
1027
|
+
gmm_prefix=("merged", "gmm"),
|
|
1028
|
+
) -> tuple[np.ndarray, CoentropyResult]:
|
|
1029
|
+
"""
|
|
1030
|
+
Parameters
|
|
1031
|
+
----------
|
|
1032
|
+
sorting : DARTsortSorting
|
|
1033
|
+
min_coentropy : float
|
|
1034
|
+
Must be met by pair for mask=True
|
|
1035
|
+
min_coverage : float
|
|
1036
|
+
Pairs such that at least one unit in each pair has
|
|
1037
|
+
rival_count/count > mincov are allowed
|
|
1038
|
+
iou_threshold: float
|
|
1039
|
+
Pairs with rival iou > iouthresh are allowed
|
|
1040
|
+
"""
|
|
1041
|
+
c = coentropy(sorting, gmm_prefix=gmm_prefix)
|
|
1042
|
+
assert c is not None
|
|
1043
|
+
|
|
1044
|
+
mask = np.logical_or(c.cov >= coverage_threshold, c.iou >= iou_threshold)
|
|
1045
|
+
mask = np.logical_and(c.coentropy >= min_coentropy, mask)
|
|
1046
|
+
np.fill_diagonal(mask, True)
|
|
1047
|
+
return mask, c
|
|
1048
|
+
|
|
1049
|
+
|
|
1050
|
+
def coentropy(
|
|
1051
|
+
sorting: DARTsortSorting,
|
|
1052
|
+
gmm_prefix=("merged", "gmm"),
|
|
1053
|
+
) -> CoentropyResult | None:
|
|
1054
|
+
"""Calculate entropy reduction due to merging pairs."""
|
|
1055
|
+
for k in gmm_prefix:
|
|
1056
|
+
cands = getattr(sorting, f"{k}_candidates", None)
|
|
1057
|
+
resps = getattr(sorting, f"{k}_responsibilities", None)
|
|
1058
|
+
if cands is not None:
|
|
1059
|
+
assert resps is not None
|
|
1060
|
+
break
|
|
1061
|
+
else:
|
|
1062
|
+
return None
|
|
1063
|
+
|
|
1064
|
+
k = sorting.n_units
|
|
1065
|
+
resps = resps[:, : cands.shape[1]].astype(np.float64)
|
|
1066
|
+
coentropy = np.zeros((k, k))
|
|
1067
|
+
cooccurrence = np.zeros((k, k), dtype=np.int64)
|
|
1068
|
+
rival_count = np.zeros((k, k), dtype=np.int64)
|
|
1069
|
+
occurrence = np.zeros((k,), dtype=np.int64)
|
|
1070
|
+
_calc_coentropy(coentropy, cooccurrence, rival_count, occurrence, cands, resps)
|
|
1071
|
+
rival_count += rival_count.T
|
|
1072
|
+
cdiag = np.diagonal(rival_count)
|
|
1073
|
+
assert (cdiag % 2 == 0).all()
|
|
1074
|
+
np.fill_diagonal(rival_count, cdiag // 2)
|
|
1075
|
+
coentropy += coentropy.T
|
|
1076
|
+
cooccurrence += cooccurrence.T
|
|
1077
|
+
|
|
1078
|
+
# rival count diagonal is just unit top count (not exactly label count,
|
|
1079
|
+
# since it doesn't account for noise assignments)
|
|
1080
|
+
counts = np.diagonal(rival_count)
|
|
1081
|
+
counts = np.maximum(counts, 1)
|
|
1082
|
+
|
|
1083
|
+
cov = rival_count / counts
|
|
1084
|
+
cov = np.minimum(cov, cov.T)
|
|
1085
|
+
|
|
1086
|
+
# this is a disjoint union, since it's the top-label count
|
|
1087
|
+
union = counts[:, None] + counts[None, :]
|
|
1088
|
+
iou = rival_count / union
|
|
1089
|
+
|
|
1090
|
+
return CoentropyResult(
|
|
1091
|
+
coentropy=coentropy,
|
|
1092
|
+
cooccurrence=cooccurrence,
|
|
1093
|
+
rival_count=rival_count,
|
|
1094
|
+
occurrence=occurrence,
|
|
1095
|
+
cov=cov,
|
|
1096
|
+
iou=iou,
|
|
1097
|
+
)
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
@numba.njit(parallel=True)
|
|
1101
|
+
def _calc_coentropy(
|
|
1102
|
+
coentropy: np.ndarray,
|
|
1103
|
+
cooccurrence: np.ndarray,
|
|
1104
|
+
rival_count: np.ndarray,
|
|
1105
|
+
occurrence: np.ndarray,
|
|
1106
|
+
cands: np.ndarray,
|
|
1107
|
+
resps: np.ndarray,
|
|
1108
|
+
):
|
|
1109
|
+
for i in numba.prange(cands.shape[0]): # ty: ignore
|
|
1110
|
+
u = cands[i]
|
|
1111
|
+
q = resps[i]
|
|
1112
|
+
log_q = np.log(q)
|
|
1113
|
+
np.nan_to_num(log_q, copy=False, neginf=0.0)
|
|
1114
|
+
dh = q * log_q
|
|
1115
|
+
|
|
1116
|
+
ui0 = u[0]
|
|
1117
|
+
qi0 = q[0]
|
|
1118
|
+
dhi0 = dh[0]
|
|
1119
|
+
|
|
1120
|
+
occurrence[ui0] += 1
|
|
1121
|
+
rival_count[ui0, ui0] += 1
|
|
1122
|
+
|
|
1123
|
+
for j in range(1, cands.shape[1]):
|
|
1124
|
+
uj = u[j]
|
|
1125
|
+
if uj < 0:
|
|
1126
|
+
break
|
|
1127
|
+
|
|
1128
|
+
ii = min(ui0, uj)
|
|
1129
|
+
jj = max(ui0, uj)
|
|
1130
|
+
|
|
1131
|
+
occurrence[uj] += 1
|
|
1132
|
+
rival_count[ui0, uj] += 1
|
|
1133
|
+
|
|
1134
|
+
cij = cooccurrence[ii, jj] + 1
|
|
1135
|
+
cooccurrence[ii, jj] = cij
|
|
1136
|
+
|
|
1137
|
+
# change in entropy due to merging uj, uk:
|
|
1138
|
+
# subtract their current contribution, add the new contribution
|
|
1139
|
+
# we want reduction of entropy, so this is the negative of that!
|
|
1140
|
+
qij = q[j] + qi0
|
|
1141
|
+
dhij = dh[j] + dhi0
|
|
1142
|
+
if qij > 0:
|
|
1143
|
+
dhij -= qij * np.log(qij)
|
|
1144
|
+
|
|
1145
|
+
# Welford mean of -dh
|
|
1146
|
+
cur_coent = coentropy[ii, jj]
|
|
1147
|
+
coentropy[ii, jj] = cur_coent + (-dhij - cur_coent) / cij
|
|
@@ -19,7 +19,16 @@ from ..util.job_util import ensure_computation_config
|
|
|
19
19
|
from ..util.main_util import ds_save_intermediate_labels
|
|
20
20
|
from ..util.motion import MotionInfo
|
|
21
21
|
from ..util.multiprocessing_util import handle_negative_jobs
|
|
22
|
-
from . import
|
|
22
|
+
from ..util.torch_util import cleanup_and_log_gpu_usage
|
|
23
|
+
from . import (
|
|
24
|
+
agglomerate,
|
|
25
|
+
cluster_util,
|
|
26
|
+
density,
|
|
27
|
+
forward_backward,
|
|
28
|
+
kmeans,
|
|
29
|
+
mixture,
|
|
30
|
+
refine_util,
|
|
31
|
+
)
|
|
23
32
|
from .clustering_features import SimpleMatrixFeatures, StableWaveformFeatures
|
|
24
33
|
|
|
25
34
|
if TYPE_CHECKING:
|
|
@@ -292,6 +301,10 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
292
301
|
workers=-1,
|
|
293
302
|
uhdversion=False,
|
|
294
303
|
random_seed=0,
|
|
304
|
+
kmeans_cleanup=True,
|
|
305
|
+
kmeans_max_sigma: float = 5.0,
|
|
306
|
+
kmeans_iter=100,
|
|
307
|
+
device=None,
|
|
295
308
|
**kwargs,
|
|
296
309
|
):
|
|
297
310
|
super().__init__(**kwargs)
|
|
@@ -308,6 +321,10 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
308
321
|
self.workers = workers
|
|
309
322
|
self.uhdversion = uhdversion
|
|
310
323
|
self.random_seed = random_seed
|
|
324
|
+
self.kmeans_cleanup = kmeans_cleanup
|
|
325
|
+
self.kmeans_max_sigma = kmeans_max_sigma
|
|
326
|
+
self.kmeans_iter = kmeans_iter
|
|
327
|
+
self.device = device
|
|
311
328
|
|
|
312
329
|
@classmethod
|
|
313
330
|
def from_config(
|
|
@@ -335,6 +352,7 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
335
352
|
outlier_radius=clustering_cfg.outlier_radius,
|
|
336
353
|
outlier_neighbor_count=clustering_cfg.outlier_neighbor_count,
|
|
337
354
|
workers=workers,
|
|
355
|
+
device=computation_cfg.actual_device(),
|
|
338
356
|
uhdversion=uhdversion,
|
|
339
357
|
computation_cfg=computation_cfg,
|
|
340
358
|
waveform_cfg=waveform_cfg,
|
|
@@ -342,6 +360,9 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
342
360
|
save_labels_dir=save_labels_dir,
|
|
343
361
|
labels_fmt=labels_fmt,
|
|
344
362
|
sampling_cfg=clustering_cfg.sampling_cfg,
|
|
363
|
+
kmeans_cleanup=clustering_cfg.dpc_kmeans_cleanup,
|
|
364
|
+
kmeans_iter=clustering_cfg.kmeans_iter,
|
|
365
|
+
kmeans_max_sigma=clustering_cfg.gmmdpc_max_sigma,
|
|
345
366
|
)
|
|
346
367
|
|
|
347
368
|
def _cluster(
|
|
@@ -425,6 +446,19 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
425
446
|
kdtree = None
|
|
426
447
|
labels = res["labels"]
|
|
427
448
|
|
|
449
|
+
if self.kmeans_cleanup:
|
|
450
|
+
kres = kmeans.truncated_kmeans_from_labels(
|
|
451
|
+
X=X,
|
|
452
|
+
labels=labels,
|
|
453
|
+
device=self.device,
|
|
454
|
+
max_sigma=self.kmeans_max_sigma,
|
|
455
|
+
n_iter=self.kmeans_iter,
|
|
456
|
+
)
|
|
457
|
+
assert kres.labels is not None
|
|
458
|
+
labels = kres.labels.numpy(force=True)
|
|
459
|
+
del kres
|
|
460
|
+
cleanup_and_log_gpu_usage(self.computation_cfg, "DPC->kmeans")
|
|
461
|
+
|
|
428
462
|
labels = cluster_util.decrumb(
|
|
429
463
|
labels, min_size=self.remove_clusters_smaller_than, in_place=True
|
|
430
464
|
)
|
|
@@ -821,6 +855,7 @@ class AgglomerateRefinement(Refinement):
|
|
|
821
855
|
recording: BaseRecording | None,
|
|
822
856
|
motion: MotionInfo,
|
|
823
857
|
):
|
|
858
|
+
assert recording is not None
|
|
824
859
|
return agglomerate.agglomerate(
|
|
825
860
|
recording=recording,
|
|
826
861
|
sorting=sorting,
|
|
@@ -25,15 +25,6 @@ from ..util.py_util import databag
|
|
|
25
25
|
from ..util.waveform_util import single_channel_index
|
|
26
26
|
from . import cluster_util
|
|
27
27
|
|
|
28
|
-
default_clustering_features_cfg = ClusteringFeaturesConfig()
|
|
29
|
-
minimal_features_cfg = ClusteringFeaturesConfig(
|
|
30
|
-
n_main_channel_pcs=0,
|
|
31
|
-
use_amplitude=False,
|
|
32
|
-
use_signed_amplitude=False,
|
|
33
|
-
use_x=False,
|
|
34
|
-
use_z=False,
|
|
35
|
-
)
|
|
36
|
-
|
|
37
28
|
logger = get_logger(__name__)
|
|
38
29
|
|
|
39
30
|
|
|
@@ -77,8 +68,14 @@ class SimpleMatrixFeatures:
|
|
|
77
68
|
)
|
|
78
69
|
if xyza is not None:
|
|
79
70
|
x = xyza[:, 0]
|
|
71
|
+
if not _allfinite(x):
|
|
72
|
+
raise ValueError(_numbers_error_str("x", x))
|
|
80
73
|
z = xyza[:, 2]
|
|
74
|
+
if not _allfinite(z):
|
|
75
|
+
raise ValueError(_numbers_error_str("z", z))
|
|
81
76
|
z_reg = motion.correct_s(t_s, z)
|
|
77
|
+
if not _allfinite(z_reg):
|
|
78
|
+
raise ValueError(_numbers_error_str("z_reg", z_reg))
|
|
82
79
|
else:
|
|
83
80
|
x = z = z_reg = None
|
|
84
81
|
|
|
@@ -99,10 +96,14 @@ class SimpleMatrixFeatures:
|
|
|
99
96
|
amp = getattr(sorting, clustering_features_cfg.amplitudes_dataset_name)
|
|
100
97
|
if clustering_features_cfg.use_amplitude:
|
|
101
98
|
assert amp is not None
|
|
99
|
+
if not _allfinite(amp):
|
|
100
|
+
raise ValueError(_numbers_error_str("amp", amp))
|
|
102
101
|
ampft = amp.copy()
|
|
103
102
|
if clustering_features_cfg.log_transform_amplitude:
|
|
104
103
|
ampft = np.log(clustering_features_cfg.amp_log_c + ampft)
|
|
105
104
|
ampft *= clustering_features_cfg.amp_scale
|
|
105
|
+
if not _allfinite(ampft):
|
|
106
|
+
raise ValueError(_numbers_error_str("ampft", ampft))
|
|
106
107
|
features.append(ampft[:, None])
|
|
107
108
|
|
|
108
109
|
v = getattr(sorting, clustering_features_cfg.voltages_dataset_name, None)
|
|
@@ -123,6 +124,8 @@ class SimpleMatrixFeatures:
|
|
|
123
124
|
rank=clustering_features_cfg.n_main_channel_pcs,
|
|
124
125
|
dataset_name=clustering_features_cfg.pca_dataset_name,
|
|
125
126
|
)
|
|
127
|
+
if not _allfinite(pcs):
|
|
128
|
+
raise ValueError(_numbers_error_str("No motion pcs", pcs))
|
|
126
129
|
elif do_pcs and clustering_features_cfg.motion_aware:
|
|
127
130
|
shifts, n_pitches_shift = motion.pitch_shifts(
|
|
128
131
|
sorting=sorting,
|
|
@@ -141,6 +144,8 @@ class SimpleMatrixFeatures:
|
|
|
141
144
|
mask = np.broadcast_to(mask, len(schan))
|
|
142
145
|
if hasattr(sorting, clustering_features_cfg.pca_dataset_name):
|
|
143
146
|
pcs = getattr(sorting, clustering_features_cfg.pca_dataset_name)
|
|
147
|
+
if not _allfinite(pcs):
|
|
148
|
+
raise ValueError(_numbers_error_str("sorting pcs", pcs))
|
|
144
149
|
erp, pcs = interpolate_by_chunk(
|
|
145
150
|
mask=mask,
|
|
146
151
|
dataset=pcs,
|
|
@@ -153,6 +158,8 @@ class SimpleMatrixFeatures:
|
|
|
153
158
|
params=clustering_features_cfg.interp_params,
|
|
154
159
|
)
|
|
155
160
|
pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
|
|
161
|
+
if not _allfinite(pcs):
|
|
162
|
+
raise ValueError(_numbers_error_str("sorting interp pcs", pcs))
|
|
156
163
|
else:
|
|
157
164
|
assert sorting.parent_h5_path is not None
|
|
158
165
|
with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
|
|
@@ -168,20 +175,25 @@ class SimpleMatrixFeatures:
|
|
|
168
175
|
params=clustering_features_cfg.interp_params,
|
|
169
176
|
)
|
|
170
177
|
pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
|
|
178
|
+
if not _allfinite(pcs):
|
|
179
|
+
raise ValueError(_numbers_error_str("h5 interp pcs", pcs))
|
|
171
180
|
|
|
172
181
|
if do_pcs:
|
|
173
182
|
assert pcs is not None
|
|
174
|
-
|
|
183
|
+
pctf = clustering_features_cfg.pc_transform
|
|
184
|
+
if pctf == "log":
|
|
175
185
|
pcs = signed_log1p(
|
|
176
186
|
pcs, pre_scale=clustering_features_cfg.pc_pre_transform_scale
|
|
177
187
|
)
|
|
178
|
-
elif
|
|
188
|
+
elif pctf == "sqrt":
|
|
179
189
|
pcs = signed_sqrt_transform(
|
|
180
190
|
pcs, pre_scale=clustering_features_cfg.pc_pre_transform_scale
|
|
181
191
|
)
|
|
182
192
|
else:
|
|
183
|
-
assert
|
|
193
|
+
assert pctf in ("none", None)
|
|
184
194
|
pcs *= clustering_features_cfg.pc_scale
|
|
195
|
+
if not _allfinite(pcs):
|
|
196
|
+
raise ValueError(_numbers_error_str(f"{pctf} pcs", pcs))
|
|
185
197
|
if torch.is_tensor(pcs):
|
|
186
198
|
pcs = pcs.numpy(force=True)
|
|
187
199
|
features.append(pcs)
|
|
@@ -310,3 +322,16 @@ def signed_sqrt_transform(x, pre_scale=1.0):
|
|
|
310
322
|
xx.sub_(1.0)
|
|
311
323
|
xx.mul_(torch.sign(x))
|
|
312
324
|
return xx
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _allfinite(x):
|
|
328
|
+
if isinstance(x, torch.Tensor):
|
|
329
|
+
return x.isfinite().all()
|
|
330
|
+
else:
|
|
331
|
+
return np.isfinite(x).all()
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _numbers_error_str(name: str, x: np.ndarray):
|
|
335
|
+
if isinstance(x, torch.Tensor):
|
|
336
|
+
x = x.numpy(force=True)
|
|
337
|
+
return f"{name}: {np.isposinf(x).sum()} +inf, {np.isneginf(x).sum()} -inf, {np.isnan(x).sum()} nan."
|