DARTsort 0.5.11__tar.gz → 0.5.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.11 → dartsort-0.5.12}/PKG-INFO +1 -1
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/agglomerate.py +4 -2
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/kmeans.py +1 -7
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/mixture.py +68 -41
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/refine_util.py +38 -141
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/config.py +2 -2
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/analysis.py +1 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/simkit.py +24 -15
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/decollider.py +2 -2
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/reduction.py +0 -1
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/data_util.py +13 -7
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/internal_config.py +1 -1
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/motion.py +10 -1
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/multiprocessing_util.py +1 -1
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/registration_util.py +13 -2
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/analysis_plots.py +1 -1
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/vismain.py +0 -2
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_dartsort.py +3 -2
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_simkit.py +1 -1
- {dartsort-0.5.11 → dartsort-0.5.12}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/.gitignore +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/README.md +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/docs/index.md +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/docs/main_api.md +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/environment.yml +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/.keep +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/basic_integration_test.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/ci diag.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/dartsort_dredge_demo.ipynb +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/dartsort_dredge_demo.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/data test.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/dead reckoning.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/denoise np1.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/denoise np2.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/exploratory.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/geom debug.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/hdbscan_clustering_5min.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/hybrid 10.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/hybrid cluster.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/hybrid_11.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/initial training.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/make hybrid.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/mgh.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/np1 feats.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/np2 processing version c.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/paperfigs.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/pdists.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/postcheck.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/ptp vae.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/re clust.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/reg fig loop.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/reg_sim_exp.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/relocating.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/relocating_all.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/restats +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/sortStinger1.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/sortStinger3.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/sortStinger4.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/sortStinger5.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/sortStinger6.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/spike farm.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/ss_ensemble_sim.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/ss_ensemble_sim_drift.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/ss_sim.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/stan diagnostics.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/storytelling c.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/storytelling.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/debug_matching.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_agreement.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_amplitude_scaling.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_batch_optim.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_ibme.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_mm_initialization.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_normxcorr1d.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_reassignment.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/tests/test_relocation.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/notebook/thomas.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/pyproject.toml +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/setup.cfg +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/__init__.py +2 -2
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/cluster_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/clustering.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/clustering_features.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/clustering/modes.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/comparison.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/main.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/peel_base.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/peel_lib.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/subtract.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/mixture_classifier.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/pipeline.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/transform_base.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/main_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/noise_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/spiketorch.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/mixture.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/SOURCES.txt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/conftest.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_alignment.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_clustering.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_config.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_data_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_density.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_detect.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_linalg.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_matching.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_modeling.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_subtract.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_templates.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_threshold.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_transform.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.11 → dartsort-0.5.12}/zensical.toml +0 -0
|
@@ -211,7 +211,7 @@ def agglomerate(
|
|
|
211
211
|
agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
|
|
212
212
|
|
|
213
213
|
agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
|
|
214
|
-
new_ids = reorder[new_ids]
|
|
214
|
+
new_ids = reorder[np.unique(new_ids, return_inverse=True)[1]]
|
|
215
215
|
|
|
216
216
|
return Agglomeration(
|
|
217
217
|
agglomerated_sorting=agg_sorting,
|
|
@@ -957,7 +957,9 @@ def _dedup_unit_loop(
|
|
|
957
957
|
score_slice = scores[i0 : i1 + 1]
|
|
958
958
|
dt_slice = dt[i0:i1]
|
|
959
959
|
order = np.argsort(score_slice)
|
|
960
|
-
|
|
960
|
+
# putting the -1 there ensures at least one spike is kept
|
|
961
|
+
# that case is only relevant when 0 in dt_slice
|
|
962
|
+
for oo in order[:-1]:
|
|
961
963
|
# discard spike i0 + oo
|
|
962
964
|
ii = i0 + oo
|
|
963
965
|
discard[ii] = True
|
|
@@ -24,7 +24,6 @@ from ..util.sparse_util import (
|
|
|
24
24
|
sparse_centroid_distsq,
|
|
25
25
|
)
|
|
26
26
|
from ..util.spiketorch import spawn_torch_rg, sqeuc_cdist_known_norm
|
|
27
|
-
from ..util.torch_util import torch_compile, torch_compiler
|
|
28
27
|
|
|
29
28
|
logger = get_logger(__name__)
|
|
30
29
|
|
|
@@ -78,6 +77,7 @@ def kmeanspp(
|
|
|
78
77
|
|
|
79
78
|
simple_case = (not min_distance) and (not skip_assignment)
|
|
80
79
|
if simple_case:
|
|
80
|
+
assert assignments is not None
|
|
81
81
|
_kmeanspp_simple_loop(
|
|
82
82
|
X=X,
|
|
83
83
|
Xnormsq=Xnormsq,
|
|
@@ -475,19 +475,16 @@ def kmeans(
|
|
|
475
475
|
)
|
|
476
476
|
|
|
477
477
|
|
|
478
|
-
@torch.compiler.disable
|
|
479
478
|
def _uniform(gen: torch.Generator, buf: Tensor):
|
|
480
479
|
return buf.uniform_(generator=gen)
|
|
481
480
|
|
|
482
481
|
|
|
483
|
-
@torch_compiler(fullgraph=False)
|
|
484
482
|
def _one_gumbel_nolog(p: Tensor, gen: torch.Generator, buf: Tensor):
|
|
485
483
|
z = _uniform(gen, buf)
|
|
486
484
|
z = z.log_()
|
|
487
485
|
return torch.divide(p, z, out=z).argmin(dim=0)
|
|
488
486
|
|
|
489
487
|
|
|
490
|
-
@torch_compile
|
|
491
488
|
def _sqeuc(X: Tensor, Xnormsq: Tensor, Y: Tensor, out: Tensor):
|
|
492
489
|
torch.addmm(Xnormsq, X, Y.transpose(0, 1), alpha=-2.0, out=out)
|
|
493
490
|
Ynormsq = (Y**2).sum(dim=1)
|
|
@@ -495,7 +492,6 @@ def _sqeuc(X: Tensor, Xnormsq: Tensor, Y: Tensor, out: Tensor):
|
|
|
495
492
|
return out
|
|
496
493
|
|
|
497
494
|
|
|
498
|
-
@torch_compiler(fullgraph=False)
|
|
499
495
|
def _kmeanspp_simple_loop(
|
|
500
496
|
*,
|
|
501
497
|
X: Tensor,
|
|
@@ -531,7 +527,6 @@ def _kmeanspp_simple_loop(
|
|
|
531
527
|
torch.minimum(dists, newdists, out=dists)
|
|
532
528
|
|
|
533
529
|
|
|
534
|
-
@torch_compiler(fullgraph=False)
|
|
535
530
|
def _kmeanspp_noassign_loop(
|
|
536
531
|
*,
|
|
537
532
|
X: Tensor,
|
|
@@ -563,7 +558,6 @@ def _kmeanspp_noassign_loop(
|
|
|
563
558
|
torch.minimum(dists, newdists, out=dists)
|
|
564
559
|
|
|
565
560
|
|
|
566
|
-
@torch_compiler(fullgraph=False)
|
|
567
561
|
def _kmeans_main_loop(
|
|
568
562
|
n_iter: int,
|
|
569
563
|
e: Tensor,
|
|
@@ -107,8 +107,8 @@ if TYPE_CHECKING:
|
|
|
107
107
|
|
|
108
108
|
logger = get_logger(__name__)
|
|
109
109
|
pnoid = logger.isEnabledFor(DARTSORTVERBOSE)
|
|
110
|
-
prop_check_atol =
|
|
111
|
-
prop_check_rtol = 1e-
|
|
110
|
+
prop_check_atol = 1e-3
|
|
111
|
+
prop_check_rtol = 1e-5
|
|
112
112
|
|
|
113
113
|
|
|
114
114
|
# -- main
|
|
@@ -1049,7 +1049,7 @@ class DenseSpikeData:
|
|
|
1049
1049
|
"""
|
|
1050
1050
|
unit_ids = torch.as_tensor(unit_ids, device=self.neighborhood_ids.device)
|
|
1051
1051
|
covered = self.lut_coverage(unit_ids, lut)
|
|
1052
|
-
candidates = torch.where(covered, unit_ids[None, :], -1)
|
|
1052
|
+
candidates = torch.where(covered, unit_ids[None, :], -1).long()
|
|
1053
1053
|
candidate_counts = covered.sum(dim=1).cpu()
|
|
1054
1054
|
|
|
1055
1055
|
batches = []
|
|
@@ -1501,7 +1501,7 @@ class StreamingSpikeData(BatchedSpikeData):
|
|
|
1501
1501
|
assert self.max_n_total is not None
|
|
1502
1502
|
assert self.n_total == self.max_n_total
|
|
1503
1503
|
proposals = full_proposal_by_neighb(self.un_adj_lut, self.n_total)
|
|
1504
|
-
self.proposals = proposals.to(self.device)
|
|
1504
|
+
self.proposals = proposals.to(self.device, dtype=torch.long)
|
|
1505
1505
|
|
|
1506
1506
|
def _update_sizes_from_n_units(self, n_units: int):
|
|
1507
1507
|
if self.max_n_total is None or self.max_n_explore is None:
|
|
@@ -1657,6 +1657,7 @@ class TruncatedSpikeData(BatchedSpikeData):
|
|
|
1657
1657
|
explore_neighb_steps: int = 1,
|
|
1658
1658
|
neighb_overlap: float = 0.75,
|
|
1659
1659
|
batch_size: int = 128,
|
|
1660
|
+
allow_gaps=False,
|
|
1660
1661
|
) -> Self:
|
|
1661
1662
|
assert len(x) == len(neighborhoods.b.neighborhood_ids)
|
|
1662
1663
|
xt, whitenedx, CmoCooinvx, noise_logliks = _whiten_impute_and_noise_score(
|
|
@@ -1685,9 +1686,11 @@ class TruncatedSpikeData(BatchedSpikeData):
|
|
|
1685
1686
|
)
|
|
1686
1687
|
assert self.candidates is not None
|
|
1687
1688
|
self.candidates[:, 0] = labels
|
|
1688
|
-
if pnoid:
|
|
1689
|
-
|
|
1690
|
-
|
|
1689
|
+
if pnoid and not allow_gaps:
|
|
1690
|
+
luniq = labels.unique().cpu()
|
|
1691
|
+
luniq = luniq[luniq >= 0]
|
|
1692
|
+
assert luniq.numel() > 0, str(luniq.tolist())
|
|
1693
|
+
assert luniq.numel() == luniq.amax() + 1, str(luniq.tolist())
|
|
1691
1694
|
return self
|
|
1692
1695
|
|
|
1693
1696
|
def update(
|
|
@@ -1708,7 +1711,7 @@ class TruncatedSpikeData(BatchedSpikeData):
|
|
|
1708
1711
|
self.un_adj_lut.b.lut,
|
|
1709
1712
|
(0, 0, 0, 1),
|
|
1710
1713
|
value=self.un_adj_lut.b.unit_ids.shape[0],
|
|
1711
|
-
)
|
|
1714
|
+
).long()
|
|
1712
1715
|
top_lut_ixs = lut_padded[
|
|
1713
1716
|
self.candidates[:, : self.n_candidates], self.neighborhood_ids[:, None]
|
|
1714
1717
|
]
|
|
@@ -3834,7 +3837,7 @@ class TruncatedMixtureModel(BaseMixtureModel):
|
|
|
3834
3837
|
assert n_new_units == sum(lp.numel() for lp in new_log_props)
|
|
3835
3838
|
if self.signal_rank:
|
|
3836
3839
|
assert all(b is not None for b in new_bases)
|
|
3837
|
-
assert n_new_units == sum(b.shape[0] for b in new_bases) # type: ignore
|
|
3840
|
+
assert n_new_units == sum(b.shape[0] for b in new_bases) # type: ignore # ty: ignore[x]
|
|
3838
3841
|
|
|
3839
3842
|
# resize params to allow space for the new guys
|
|
3840
3843
|
Korig = self.n_units
|
|
@@ -4243,6 +4246,7 @@ def get_truncated_datasets(
|
|
|
4243
4246
|
neighb_cov=neighb_cov,
|
|
4244
4247
|
seed=rg,
|
|
4245
4248
|
batch_size=refinement_cfg.eval_batch_size,
|
|
4249
|
+
allow_gaps=True,
|
|
4246
4250
|
)
|
|
4247
4251
|
assert torch.equal(
|
|
4248
4252
|
val_data.neighborhoods.b.neighborhoods,
|
|
@@ -4427,6 +4431,8 @@ def instantiate_and_bootstrap_tmm(
|
|
|
4427
4431
|
sorting, min_spikes=refinement_cfg.min_count
|
|
4428
4432
|
)
|
|
4429
4433
|
sorting = sorting.flatten()
|
|
4434
|
+
if not sorting.n_units:
|
|
4435
|
+
raise ValueError("No units to cluster.")
|
|
4430
4436
|
|
|
4431
4437
|
neighb_cov, erp, train_data, val_data, full_data, noise, train_ixs, val_ixs = (
|
|
4432
4438
|
get_truncated_datasets(
|
|
@@ -5554,7 +5560,10 @@ def _evaluate_single_demolition(
|
|
|
5554
5560
|
|
|
5555
5561
|
# evaluate effect on heldout set
|
|
5556
5562
|
eval_scores_adj = proportion_adjust_scores(
|
|
5557
|
-
scores=eval_scores,
|
|
5563
|
+
scores=eval_scores,
|
|
5564
|
+
orig_log_props=orig_log_props,
|
|
5565
|
+
new_log_props=new_log_props,
|
|
5566
|
+
sort=False,
|
|
5558
5567
|
)
|
|
5559
5568
|
return ecl(
|
|
5560
5569
|
resps=eval_scores_adj.responsibilities,
|
|
@@ -6070,7 +6079,7 @@ def combine_luts(*luts: NeighborhoodLUT) -> NeighborhoodLUT:
|
|
|
6070
6079
|
return luts[0]
|
|
6071
6080
|
adj = luts[0].b.lut < luts[0].b.unit_ids.shape[0]
|
|
6072
6081
|
for ll in luts[1:]:
|
|
6073
|
-
adj.logical_or_(ll.lut < ll.unit_ids.shape[0]) # type: ignore
|
|
6082
|
+
adj.logical_or_(ll.lut < ll.unit_ids.shape[0]) # type: ignore # ty: ignore[x]
|
|
6074
6083
|
unit_ids, neighb_ids = adj.nonzero(as_tuple=True)
|
|
6075
6084
|
lut = torch.full_like(luts[0].b.lut, unit_ids.shape[0])
|
|
6076
6085
|
lut[unit_ids, neighb_ids] = torch.arange(unit_ids.shape[0], device=lut.device)
|
|
@@ -6106,7 +6115,7 @@ def candidate_search_sets(
|
|
|
6106
6115
|
tops, topunits = torch.topk(s, k=n_search, dim=1)
|
|
6107
6116
|
topunits.masked_fill_(tops == 0, -1)
|
|
6108
6117
|
# pad with row of -1s for the invalid lut ixs (===n_lut)
|
|
6109
|
-
topunits = F.pad(topunits, (0, 0, 0, 1), value=-1)
|
|
6118
|
+
topunits = F.pad(topunits, (0, 0, 0, 1), value=-1).long()
|
|
6110
6119
|
|
|
6111
6120
|
return topunits
|
|
6112
6121
|
|
|
@@ -6120,7 +6129,7 @@ def full_proposal_by_neighb(lut: NeighborhoodLUT, max_proposed: int):
|
|
|
6120
6129
|
"""Returns n_neighbs x max possible proposed units array, basically transposing the LUT's nonzeros."""
|
|
6121
6130
|
n_neighbs = lut.b.lut.shape[1]
|
|
6122
6131
|
n_lut = lut.b.unit_ids.shape[0]
|
|
6123
|
-
proposals = torch.full((n_neighbs, max_proposed), -1)
|
|
6132
|
+
proposals = torch.full((n_neighbs, max_proposed), -1, dtype=torch.long)
|
|
6124
6133
|
for neighb_id in range(n_neighbs):
|
|
6125
6134
|
row = lut.b.lut[:, neighb_id]
|
|
6126
6135
|
(row_valid,) = (row < n_lut).nonzero(as_tuple=True)
|
|
@@ -6157,13 +6166,13 @@ def _fill_blank_labels(
|
|
|
6157
6166
|
|
|
6158
6167
|
# need full coverage of neighborhoods here. would prefer only to branch out with
|
|
6159
6168
|
# explore as needed, so make those probs tiny.
|
|
6160
|
-
keep_same_adj = un_adj.sum(0).
|
|
6169
|
+
keep_same_adj = un_adj.sum(0).amin().cpu().item() > 0
|
|
6161
6170
|
n_new_units = 0
|
|
6162
6171
|
if keep_same_adj:
|
|
6163
6172
|
adj = un_adj
|
|
6164
6173
|
else:
|
|
6165
6174
|
adj = un_adj + explore_adj * torch.finfo(explore_adj.dtype).tiny
|
|
6166
|
-
uncovered = adj.sum(0).
|
|
6175
|
+
uncovered = adj.sum(0).amin().cpu().item() == 0
|
|
6167
6176
|
if uncovered:
|
|
6168
6177
|
uncovered_adj = adj.clone()
|
|
6169
6178
|
else:
|
|
@@ -6176,7 +6185,7 @@ def _fill_blank_labels(
|
|
|
6176
6185
|
if adj.sum(0).min().cpu().item() > 0:
|
|
6177
6186
|
break
|
|
6178
6187
|
|
|
6179
|
-
still_uncovered = adj.sum(0).
|
|
6188
|
+
still_uncovered = adj.sum(0).amin().cpu().item() == 0
|
|
6180
6189
|
if still_uncovered and ensure_coverage_only and allow_new_units:
|
|
6181
6190
|
# make new units by getting connected components of the uncovered
|
|
6182
6191
|
# neighborhood graph
|
|
@@ -6221,13 +6230,20 @@ def _fill_blank_labels(
|
|
|
6221
6230
|
if ensure_coverage_only:
|
|
6222
6231
|
return None, n_new_units
|
|
6223
6232
|
|
|
6233
|
+
if pnoid:
|
|
6234
|
+
assert adj.amin() >= 0
|
|
6235
|
+
|
|
6224
6236
|
for i0 in range(0, Nblank, batch_size):
|
|
6225
6237
|
i1 = min(Nblank, i0 + batch_size)
|
|
6226
6238
|
ii = blank[i0:i1]
|
|
6227
6239
|
nn = neighborhood_ids[ii]
|
|
6228
|
-
p = adj[
|
|
6240
|
+
p = adj.T[nn]
|
|
6241
|
+
p_is_0 = p.sum(1) <= 0.0
|
|
6242
|
+
p[:, 0].clamp_min_(p_is_0.to(p))
|
|
6229
6243
|
draws = torch.multinomial(p, 1, replacement=True, generator=gen)
|
|
6230
|
-
|
|
6244
|
+
draws = draws.view(ii.shape[0])
|
|
6245
|
+
draws.masked_fill_(p_is_0, -1)
|
|
6246
|
+
labels[ii] = draws
|
|
6231
6247
|
|
|
6232
6248
|
return keep_same_adj, n_new_units
|
|
6233
6249
|
|
|
@@ -6306,8 +6322,10 @@ def _get_explore_sampling_data(
|
|
|
6306
6322
|
p.masked_fill_(pzero, torch.finfo(p.dtype).tiny)
|
|
6307
6323
|
|
|
6308
6324
|
# pad p with an extra LUT index (row), inds with a -1
|
|
6309
|
-
|
|
6310
|
-
|
|
6325
|
+
# the tiny mops up cases with no overlaps, and that case is
|
|
6326
|
+
# sometimes allowed by the logic which gets us here
|
|
6327
|
+
p = F.pad(p, (0, 0, 0, 1), value=torch.finfo(p.dtype).tiny)
|
|
6328
|
+
inds = F.pad(inds, (0, 0, 0, 1), value=-1).long()
|
|
6311
6329
|
|
|
6312
6330
|
return p, inds
|
|
6313
6331
|
|
|
@@ -6350,8 +6368,10 @@ def _dedup_candidates(candidates):
|
|
|
6350
6368
|
|
|
6351
6369
|
def _count_candidates(candidates, batch_candidate_counts, batch_size):
|
|
6352
6370
|
counts = candidates.new_zeros(batch_candidate_counts.shape)
|
|
6353
|
-
|
|
6354
|
-
|
|
6371
|
+
batch_iter = range(0, candidates.shape[0], batch_size)
|
|
6372
|
+
assert counts.shape[0] == len(batch_iter)
|
|
6373
|
+
for b, i0 in enumerate(batch_iter):
|
|
6374
|
+
counts[b] = (candidates[i0 : i0 + batch_size] >= 0).count_nonzero()
|
|
6355
6375
|
batch_candidate_counts.copy_(counts.cpu())
|
|
6356
6376
|
|
|
6357
6377
|
|
|
@@ -6395,38 +6415,47 @@ def concatenate_scores(scoress: list[Scores], dim=0) -> Scores:
|
|
|
6395
6415
|
)
|
|
6396
6416
|
|
|
6397
6417
|
|
|
6398
|
-
def remove_units_from_scores(scores: Scores, unit_ids: Tensor) -> Scores:
|
|
6418
|
+
def remove_units_from_scores(scores: Scores, unit_ids: Tensor, sort=True) -> Scores:
|
|
6399
6419
|
"""Return copy of scores where log_liks for candidates in unit_ids are -inf (and resps are 0)."""
|
|
6400
6420
|
bye = torch.isin(scores.candidates, unit_ids.to(scores.candidates))
|
|
6401
6421
|
new_cand = scores.candidates.masked_fill(bye, -1)
|
|
6402
6422
|
new_log_lik = scores.log_liks.clone()
|
|
6403
6423
|
new_log_lik[:, : new_cand.shape[1]].masked_fill_(bye, -torch.inf)
|
|
6404
|
-
new_resp = new_log_lik.softmax(dim=1)
|
|
6405
|
-
|
|
6424
|
+
new_resp = new_log_lik.softmax(dim=1).nan_to_num_()
|
|
6425
|
+
scores = Scores(
|
|
6406
6426
|
candidates=new_cand,
|
|
6407
6427
|
log_liks=new_log_lik,
|
|
6408
6428
|
responsibilities=new_resp,
|
|
6409
6429
|
duties=scores.duties,
|
|
6410
6430
|
)
|
|
6431
|
+
if sort:
|
|
6432
|
+
return ensure_sorted_scores(scores)
|
|
6433
|
+
else:
|
|
6434
|
+
return scores
|
|
6411
6435
|
|
|
6412
6436
|
|
|
6413
6437
|
def proportion_adjust_scores(
|
|
6414
|
-
scores: Scores, orig_log_props: Tensor, new_log_props: Tensor
|
|
6438
|
+
scores: Scores, orig_log_props: Tensor, new_log_props: Tensor, sort=True
|
|
6415
6439
|
) -> Scores:
|
|
6416
6440
|
"""What would the scores be if the log props were these, not those?"""
|
|
6417
6441
|
diff = new_log_props - orig_log_props
|
|
6442
|
+
diff.nan_to_num_()
|
|
6418
6443
|
diff = F.pad(diff, (0, 1))
|
|
6419
6444
|
diff = diff[scores.candidates]
|
|
6420
6445
|
new_log_liks = scores.log_liks.clone()
|
|
6421
6446
|
if scores.duties is not None:
|
|
6422
6447
|
diff *= scores.duties[:, None]
|
|
6423
6448
|
new_log_liks[:, : diff.shape[1]] += diff
|
|
6424
|
-
|
|
6449
|
+
scores = Scores(
|
|
6425
6450
|
log_liks=new_log_liks,
|
|
6426
6451
|
candidates=scores.candidates,
|
|
6427
|
-
responsibilities=new_log_liks.softmax(dim=1),
|
|
6452
|
+
responsibilities=new_log_liks.softmax(dim=1).nan_to_num_(),
|
|
6428
6453
|
duties=scores.duties,
|
|
6429
6454
|
)
|
|
6455
|
+
if sort:
|
|
6456
|
+
return ensure_sorted_scores(scores)
|
|
6457
|
+
else:
|
|
6458
|
+
return scores
|
|
6430
6459
|
|
|
6431
6460
|
|
|
6432
6461
|
def ensure_sorted_scores(scores: Scores) -> Scores:
|
|
@@ -6443,7 +6472,7 @@ def ensure_sorted_scores(scores: Scores) -> Scores:
|
|
|
6443
6472
|
assert scores.log_liks.shape[1] == C + 1
|
|
6444
6473
|
log_liks[:, -1] = scores.log_liks[:, -1]
|
|
6445
6474
|
if scores.responsibilities is not None:
|
|
6446
|
-
responsibilities = F.softmax(log_liks, dim=1)
|
|
6475
|
+
responsibilities = F.softmax(log_liks, dim=1).nan_to_num_()
|
|
6447
6476
|
else:
|
|
6448
6477
|
responsibilities = None
|
|
6449
6478
|
|
|
@@ -6472,18 +6501,14 @@ def drop_units_and_update_scores(
|
|
|
6472
6501
|
new_prop *= model_prop / new_prop.sum()
|
|
6473
6502
|
orig_log_prop = orig_prop.log()
|
|
6474
6503
|
new_log_prop = new_prop.log()
|
|
6475
|
-
updated = torch.logical_not(torch.isclose(orig_log_prop, new_log_prop))
|
|
6476
|
-
updated[remove_ids] = False
|
|
6477
6504
|
|
|
6478
6505
|
# update the train scores
|
|
6479
6506
|
train_scores = proportion_adjust_scores(train_scores, orig_log_prop, new_log_prop)
|
|
6480
|
-
train_scores = ensure_sorted_scores(train_scores)
|
|
6481
6507
|
|
|
6482
6508
|
# update other scores
|
|
6483
6509
|
if scores is not None:
|
|
6484
|
-
scores = remove_units_from_scores(scores, remove_ids)
|
|
6510
|
+
scores = remove_units_from_scores(scores, remove_ids, sort=False)
|
|
6485
6511
|
scores = proportion_adjust_scores(scores, orig_log_prop, new_log_prop)
|
|
6486
|
-
scores = ensure_sorted_scores(scores)
|
|
6487
6512
|
|
|
6488
6513
|
return train_scores, scores
|
|
6489
6514
|
|
|
@@ -6511,7 +6536,6 @@ def mean_responsibilities(
|
|
|
6511
6536
|
cand = scores.candidates
|
|
6512
6537
|
else:
|
|
6513
6538
|
assert False
|
|
6514
|
-
|
|
6515
6539
|
assert resp is not None
|
|
6516
6540
|
assert cand is not None
|
|
6517
6541
|
ncand = cand.shape[1]
|
|
@@ -6525,7 +6549,7 @@ def mean_responsibilities(
|
|
|
6525
6549
|
assert n_units > 0
|
|
6526
6550
|
|
|
6527
6551
|
# count candidates per batch
|
|
6528
|
-
ncand = (cand >= 0).
|
|
6552
|
+
ncand = (cand >= 0).count_nonzero(dim=1)
|
|
6529
6553
|
padlen = batch_size * int(math.ceil(cand.shape[0] / batch_size))
|
|
6530
6554
|
if padlen > ncand.shape[0]:
|
|
6531
6555
|
ncand = F.pad(ncand, (0, padlen - ncand.shape[0]))
|
|
@@ -6538,14 +6562,16 @@ def mean_responsibilities(
|
|
|
6538
6562
|
for bix, i0 in enumerate(range(0, resp.shape[0], batch_size)):
|
|
6539
6563
|
i1 = min(resp.shape[0], i0 + batch_size)
|
|
6540
6564
|
nbatch = i1 - i0
|
|
6565
|
+
if not nbatch:
|
|
6566
|
+
continue
|
|
6541
6567
|
rsum_batch.zero_()
|
|
6542
6568
|
|
|
6543
6569
|
nc = int(ncand[bix].item())
|
|
6544
6570
|
cii, cjj = _nonzero_static(cand[i0:i1] >= 0, size=nc).T
|
|
6545
|
-
|
|
6546
|
-
|
|
6547
|
-
|
|
6548
|
-
|
|
6571
|
+
if cii.numel():
|
|
6572
|
+
c = cand[i0:i1][cii, cjj].long()
|
|
6573
|
+
r = resp[i0:i1][cii, cjj].double()
|
|
6574
|
+
rsum_batch.scatter_add_(dim=0, index=c, src=r)
|
|
6549
6575
|
if includes_noise:
|
|
6550
6576
|
rsum_batch[n_units] += resp[i0:i1, -1].double().sum()
|
|
6551
6577
|
|
|
@@ -6554,7 +6580,8 @@ def mean_responsibilities(
|
|
|
6554
6580
|
|
|
6555
6581
|
# check that things didn't explode
|
|
6556
6582
|
assert resp_mean.isfinite().all(), "Responsibility mean not finite"
|
|
6557
|
-
assert torch.all(resp_mean >=
|
|
6583
|
+
assert torch.all(resp_mean >= -prop_check_atol), "Responsiblity mean negative"
|
|
6584
|
+
resp_mean.relu_()
|
|
6558
6585
|
assert torch.all(resp_mean <= 1.0 + prop_check_atol), (
|
|
6559
6586
|
f"Responsiblity mean > 1, largest: {resp_mean.amax()}"
|
|
6560
6587
|
)
|
|
@@ -271,35 +271,39 @@ def collision_cleaning_error_filter(
|
|
|
271
271
|
# discard bad units
|
|
272
272
|
keep_mask = np.ones(nu0, dtype=np.bool)
|
|
273
273
|
bad_ids = np.flatnonzero(dist > refinement_cfg.collision_cleaning_error_threshold)
|
|
274
|
+
logger.dartsortdebug(
|
|
275
|
+
f"Collision-cleaning error filter drops {bad_ids.size} / {nu0} units."
|
|
276
|
+
)
|
|
274
277
|
keep_mask[bad_ids] = False
|
|
275
278
|
spike_keep_mask = keep_mask[sorting.labels]
|
|
276
279
|
new_labels = np.where(spike_keep_mask, sorting.labels, -1)
|
|
277
280
|
|
|
278
281
|
new_props = dict(labels=new_labels)
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
282
|
+
if bad_ids.size:
|
|
283
|
+
try:
|
|
284
|
+
scores = get_gmm_scores(sorting, prefixes=["gmm"])
|
|
285
|
+
scores, _ = drop_units_and_update_scores(
|
|
286
|
+
train_scores=scores,
|
|
287
|
+
scores=None,
|
|
288
|
+
n_units=nu0,
|
|
289
|
+
remove_ids=torch.tensor(bad_ids, dtype=torch.long),
|
|
290
|
+
)
|
|
287
291
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
292
|
+
# but also fully delete those spikes.
|
|
293
|
+
cand = scores.candidates.numpy(force=True)
|
|
294
|
+
ll = scores.log_liks.numpy(force=True)
|
|
295
|
+
del_mask = np.logical_not(spike_keep_mask)
|
|
296
|
+
cand[del_mask] = -1
|
|
297
|
+
ll[del_mask, : cand.shape[1]] = -np.inf
|
|
298
|
+
new_props["gmm_candidates"] = cand
|
|
299
|
+
new_props["gmm_log_liks"] = ll
|
|
300
|
+
if scores.responsibilities is not None:
|
|
301
|
+
resp = scores.responsibilities.numpy(force=True)
|
|
302
|
+
resp[del_mask, : cand.shape[1]] = 0.0
|
|
303
|
+
resp[del_mask, -1] = 1.0
|
|
304
|
+
new_props["gmm_responsibilities"] = resp
|
|
305
|
+
except AttributeError:
|
|
306
|
+
pass
|
|
303
307
|
|
|
304
308
|
sorting = sorting.ephemeral_replace(**new_props)
|
|
305
309
|
sorting = sorting.flatten(include_gmm_properties=True)
|
|
@@ -374,6 +378,7 @@ def gmm_isolation_filter(
|
|
|
374
378
|
if ngood == gi.isolation.shape[0]:
|
|
375
379
|
break
|
|
376
380
|
|
|
381
|
+
# TODO: why is bad_guy coming through in sorted order?
|
|
377
382
|
bad_guy = np.argmax(np.where(good, -np.inf, gi.isolation))
|
|
378
383
|
removed_ids.append(bad_guy)
|
|
379
384
|
|
|
@@ -384,11 +389,17 @@ def gmm_isolation_filter(
|
|
|
384
389
|
update[needs_update[needs_update >= 0]] = True
|
|
385
390
|
update[good] = False
|
|
386
391
|
|
|
392
|
+
logger.dartsortverbose(
|
|
393
|
+
f"Drop {bad_guy} (%s / %s total, %s good).",
|
|
394
|
+
len(removed_ids),
|
|
395
|
+
update.size,
|
|
396
|
+
ngood,
|
|
397
|
+
)
|
|
387
398
|
scores, _ = drop_units_and_update_scores(
|
|
388
399
|
train_scores=scores,
|
|
389
400
|
scores=None,
|
|
390
401
|
n_units=unit_ids.shape[0],
|
|
391
|
-
remove_ids=torch.tensor([bad_guy]),
|
|
402
|
+
remove_ids=torch.tensor([bad_guy], dtype=torch.long),
|
|
392
403
|
)
|
|
393
404
|
|
|
394
405
|
keep_mask = np.ones(unit_ids.max() + 1)
|
|
@@ -565,6 +576,8 @@ def _iso_job(unit_id) -> tuple[float, np.ndarray | None, np.ndarray | None]:
|
|
|
565
576
|
try:
|
|
566
577
|
# select bandwidth based on positive part
|
|
567
578
|
bw = improved_sheather_jones(lr[:, None])
|
|
579
|
+
if not np.isfinite(bw):
|
|
580
|
+
return np.nan, None, None
|
|
568
581
|
# reweight near 0
|
|
569
582
|
weights = norm.sf(0, loc=lr, scale=bw)
|
|
570
583
|
# reflect and stack
|
|
@@ -591,8 +604,7 @@ def _iso_job(unit_id) -> tuple[float, np.ndarray | None, np.ndarray | None]:
|
|
|
591
604
|
|
|
592
605
|
# find the HIGHEST local max right of 0, if any
|
|
593
606
|
peak_ix_right = _find_right_peak_index(ev, 1)
|
|
594
|
-
|
|
595
|
-
no_right_peak = peak_ix_right == ev.shape[0] or ev0 > evpeak
|
|
607
|
+
no_right_peak = peak_ix_right == ev.shape[0] or ev0 > ev[peak_ix_right]
|
|
596
608
|
|
|
597
609
|
if no_right_peak:
|
|
598
610
|
# no isolation
|
|
@@ -621,121 +633,6 @@ def _iso_job(unit_id) -> tuple[float, np.ndarray | None, np.ndarray | None]:
|
|
|
621
633
|
return iso, domain, kde
|
|
622
634
|
|
|
623
635
|
|
|
624
|
-
# def _iso_job(unit_id) -> tuple[float, np.ndarray | None, np.ndarray | None]:
|
|
625
|
-
# p = cast(_GMMIsolationContext, _iso_ctx.ctx)
|
|
626
|
-
# assert p is not None
|
|
627
|
-
|
|
628
|
-
# # top indices
|
|
629
|
-
# (inu,) = (p.cand[:, 0] == unit_id).nonzero(as_tuple=True)
|
|
630
|
-
# if inu.numel() <= p.min_count:
|
|
631
|
-
# return np.nan, None, None
|
|
632
|
-
|
|
633
|
-
# # second best indices
|
|
634
|
-
# (byu,) = (p.cand[:, 1] == unit_id).nonzero(as_tuple=True)
|
|
635
|
-
|
|
636
|
-
# # likelihood ratio when unit comes first vs second
|
|
637
|
-
# in_ll = p.log_liks[inu, :-1]
|
|
638
|
-
# # in_denom = in_ll[:, 1:].logsumexp(dim=1)
|
|
639
|
-
# in_denom = in_ll[:, 1]
|
|
640
|
-
# min_denom, max_denom = in_denom.aminmax()
|
|
641
|
-
# in_lr = in_ll[:, 0] - in_denom
|
|
642
|
-
# if torch.isneginf(max_denom):
|
|
643
|
-
# # this unit is super isolated.
|
|
644
|
-
# return 0.0, None, None
|
|
645
|
-
# if torch.isneginf(min_denom):
|
|
646
|
-
# # first check that enough are finite
|
|
647
|
-
# (finite,) = torch.isfinite(in_lr).nonzero(as_tuple=True)
|
|
648
|
-
# nfinite = finite.numel()
|
|
649
|
-
# if nfinite < p.min_count:
|
|
650
|
-
# return np.nan, None, None
|
|
651
|
-
# if nfinite / inu.size < p.neighbor_fraction:
|
|
652
|
-
# return np.nan, None, None
|
|
653
|
-
|
|
654
|
-
# # handle +inf lrs by replacing with neg log liks
|
|
655
|
-
# in_lr.clamp_(max=in_ll[:, 0].abs_())
|
|
656
|
-
|
|
657
|
-
# # by_lr is finite by construction unless numerical catastrophe happens
|
|
658
|
-
# by_lr = p.log_liks[byu, :2].diff(dim=1)[:, 0]
|
|
659
|
-
# by_lr.clamp_(min=-1000.0)
|
|
660
|
-
|
|
661
|
-
# in_lr = in_lr.double()
|
|
662
|
-
# lr = torch.concatenate([by_lr, in_lr])
|
|
663
|
-
|
|
664
|
-
# # # weights to avoid being overwhelmed by the "by" category
|
|
665
|
-
# # nin = in_lr.numel()
|
|
666
|
-
# # nby = by_lr.numel()
|
|
667
|
-
# # if nby > nin:
|
|
668
|
-
# # weights = torch.empty_like(lr, device="cpu").numpy(force=True)
|
|
669
|
-
# # weights[:nby] = nin / nby
|
|
670
|
-
# # weights[nby:] = 1.0
|
|
671
|
-
# # else:
|
|
672
|
-
# # weights = None
|
|
673
|
-
# weights = None
|
|
674
|
-
|
|
675
|
-
# # fit kde. will symmetrize around 0 to avoid boundary issues (since lr>=0)
|
|
676
|
-
# amax = lr.abs().amax().item()
|
|
677
|
-
# lr = lr.numpy(force=True)
|
|
678
|
-
# try:
|
|
679
|
-
# # select bandwidth based on positive part
|
|
680
|
-
# bw = improved_sheather_jones(in_lr.numpy(force=True)[:, None])
|
|
681
|
-
# kde = FFTKDE(bw=bw).fit(lr, weights=weights)
|
|
682
|
-
# except ValueError:
|
|
683
|
-
# return np.nan, None, None
|
|
684
|
-
|
|
685
|
-
# # evaluate kde. grid needs to cover data.
|
|
686
|
-
# end = p.dx * np.ceil((amax + p.dx) / p.dx)
|
|
687
|
-
# grid = np.arange(0, end + p.dx / 2, p.dx)
|
|
688
|
-
# grid_left = -grid[1:][::-1]
|
|
689
|
-
# grid = np.concatenate((grid_left, grid))
|
|
690
|
-
# nc = grid.shape[0] // 2
|
|
691
|
-
# # nc = 0
|
|
692
|
-
# assert grid.shape == (2 * nc + 1,)
|
|
693
|
-
# assert grid[nc] == 0.0
|
|
694
|
-
# ev = kde.evaluate(grid)
|
|
695
|
-
|
|
696
|
-
# # find the HIGHEST local max right of 0, if any
|
|
697
|
-
# peak_ix_right = _find_right_peak_index(ev, nc)
|
|
698
|
-
# # find the NEAREST local max left or at 0, if any
|
|
699
|
-
# peak_ix_left = _find_left_peak_index(ev, nc)
|
|
700
|
-
# no_right_peak = peak_ix_right == ev.shape[0]
|
|
701
|
-
# no_left_peak = peak_ix_left == -1
|
|
702
|
-
# # make sure that we don't hit unreachable code in left peak
|
|
703
|
-
# assert peak_ix_left != -2
|
|
704
|
-
# # there has to be a peak. kdes can't ever be flat.
|
|
705
|
-
# assert not (no_left_peak and no_right_peak)
|
|
706
|
-
|
|
707
|
-
# if no_right_peak:
|
|
708
|
-
# # no isolation
|
|
709
|
-
# iso = 1.0
|
|
710
|
-
# elif no_left_peak:
|
|
711
|
-
# # good isolation
|
|
712
|
-
# iso = 0.0
|
|
713
|
-
# else:
|
|
714
|
-
# # main case
|
|
715
|
-
# assert peak_ix_right > peak_ix_left
|
|
716
|
-
# rad = int(np.ceil(5 / p.dx))
|
|
717
|
-
# dip = np.min(ev[max(nc - rad, peak_ix_left) : min(nc + rad + 1, peak_ix_right)])
|
|
718
|
-
# peak = ev[peak_ix_right]
|
|
719
|
-
# iso = dip / peak
|
|
720
|
-
|
|
721
|
-
# if p.return_kdes:
|
|
722
|
-
# # ev = ev[nc:]
|
|
723
|
-
# if grid[-1] < p.kde_rhs:
|
|
724
|
-
# domain = np.arange(-p.kde_rhs, p.kde_rhs + p.dx / 2, p.dx)
|
|
725
|
-
# npad = domain.size - grid.size
|
|
726
|
-
# assert npad % 2 == 0
|
|
727
|
-
# kde = np.pad(ev, (npad // 2, npad // 2))
|
|
728
|
-
# else:
|
|
729
|
-
# mask = grid == grid.clip(-p.kde_rhs, p.kde_rhs)
|
|
730
|
-
# kde = ev[mask]
|
|
731
|
-
# domain = grid[mask]
|
|
732
|
-
# else:
|
|
733
|
-
# kde = domain = None
|
|
734
|
-
|
|
735
|
-
# # compare value at 0 to peak value
|
|
736
|
-
# return iso, domain, kde
|
|
737
|
-
|
|
738
|
-
|
|
739
636
|
@numba.njit
|
|
740
637
|
def _find_right_peak_index(x: np.ndarray, i: int) -> int:
|
|
741
638
|
n = x.shape[0]
|
|
@@ -50,7 +50,7 @@ class DARTsortUserConfig:
|
|
|
50
50
|
subsampling_spikes_per_channel: int | None = 5000
|
|
51
51
|
"""Detection steps before the final matching round will run until at least
|
|
52
52
|
this many spikes are found or the whole recording is covered, to make sure
|
|
53
|
-
that there is enough data for clustering. See also
|
|
53
|
+
that there is enough data for clustering. See also subsampling_presence.
|
|
54
54
|
Set to None to disable subsampling."""
|
|
55
55
|
|
|
56
56
|
subsampling_presence: Annotated[float, Field(gt=0.0, le=1.0)] = 0.1
|
|
@@ -332,7 +332,7 @@ class DeveloperConfig(DARTsortUserConfig):
|
|
|
332
332
|
n_neighbors_search: int | None = 50
|
|
333
333
|
|
|
334
334
|
# filters
|
|
335
|
-
gmm_isolation_threshold: float | None =
|
|
335
|
+
gmm_isolation_threshold: float | None = None
|
|
336
336
|
collision_cleaning_error_threshold: float | None = 0.3
|
|
337
337
|
|
|
338
338
|
# gaussian mixture high level
|
|
@@ -287,6 +287,7 @@ class DARTsortAnalysis:
|
|
|
287
287
|
template_data=template_data,
|
|
288
288
|
motion=motion,
|
|
289
289
|
features_cfg=clustering_features_cfg,
|
|
290
|
+
compute_extensions_if_templates=None,
|
|
290
291
|
)
|
|
291
292
|
analyzer.compute("quality_metrics", metric_names=quality_metrics)
|
|
292
293
|
qc_df = analyzer.get_extension("quality_metrics").get_data()
|