DARTsort 0.5.12__tar.gz → 0.5.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.12 → dartsort-0.5.13}/PKG-INFO +1 -1
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/agglomerate.py +2 -7
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/clustering.py +22 -3
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/clustering_features.py +70 -22
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/mixture.py +1 -1
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/analysis.py +8 -1
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/comparison.py +2 -2
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/grab.py +2 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching.py +13 -1
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/peel_base.py +2 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/amortized_localization.py +3 -3
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/mixture_classifier.py +102 -72
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/temporal_pca.py +15 -3
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/data_util.py +25 -4
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/internal_config.py +2 -1
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/noise_util.py +18 -2
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/analysis_plots.py +99 -6
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/mixture.py +28 -7
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/scatterplots.py +1 -1
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/vismain.py +2 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/SOURCES.txt +1 -52
- dartsort-0.5.13/tests/test_agglomerate.py +35 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_data_util.py +1 -0
- dartsort-0.5.13/tests/test_mixture_classifier.py +138 -0
- dartsort-0.5.12/notebook/basic_integration_test.py +0 -137
- dartsort-0.5.12/notebook/ci diag.py +0 -573
- dartsort-0.5.12/notebook/dartsort_dredge_demo.ipynb +0 -437
- dartsort-0.5.12/notebook/dartsort_dredge_demo.py +0 -144
- dartsort-0.5.12/notebook/data test.py +0 -259
- dartsort-0.5.12/notebook/dead reckoning.py +0 -361
- dartsort-0.5.12/notebook/denoise np1.py +0 -356
- dartsort-0.5.12/notebook/denoise np2.py +0 -480
- dartsort-0.5.12/notebook/exploratory.py +0 -431
- dartsort-0.5.12/notebook/geom debug.py +0 -167
- dartsort-0.5.12/notebook/hdbscan_clustering_5min.py +0 -1294
- dartsort-0.5.12/notebook/hybrid 10.py +0 -1662
- dartsort-0.5.12/notebook/hybrid cluster.py +0 -475
- dartsort-0.5.12/notebook/hybrid_11.py +0 -1214
- dartsort-0.5.12/notebook/initial training.py +0 -162
- dartsort-0.5.12/notebook/make hybrid.py +0 -1376
- dartsort-0.5.12/notebook/mgh.py +0 -514
- dartsort-0.5.12/notebook/np1 feats.py +0 -1211
- dartsort-0.5.12/notebook/np2 processing version c.py +0 -321
- dartsort-0.5.12/notebook/paperfigs.py +0 -907
- dartsort-0.5.12/notebook/pdists.py +0 -641
- dartsort-0.5.12/notebook/postcheck.py +0 -803
- dartsort-0.5.12/notebook/ptp vae.py +0 -206
- dartsort-0.5.12/notebook/re clust.py +0 -893
- dartsort-0.5.12/notebook/reg fig loop.py +0 -723
- dartsort-0.5.12/notebook/reg_sim_exp.py +0 -493
- dartsort-0.5.12/notebook/relocating.py +0 -284
- dartsort-0.5.12/notebook/relocating_all.py +0 -182
- dartsort-0.5.12/notebook/restats +0 -0
- dartsort-0.5.12/notebook/sortStinger1.py +0 -1590
- dartsort-0.5.12/notebook/sortStinger3.py +0 -1541
- dartsort-0.5.12/notebook/sortStinger4.py +0 -1835
- dartsort-0.5.12/notebook/sortStinger5.py +0 -1586
- dartsort-0.5.12/notebook/sortStinger6.py +0 -1915
- dartsort-0.5.12/notebook/spike farm.py +0 -100
- dartsort-0.5.12/notebook/ss_ensemble_sim.py +0 -1965
- dartsort-0.5.12/notebook/ss_ensemble_sim_drift.py +0 -1928
- dartsort-0.5.12/notebook/ss_sim.py +0 -1965
- dartsort-0.5.12/notebook/stan diagnostics.py +0 -207
- dartsort-0.5.12/notebook/storytelling c.py +0 -686
- dartsort-0.5.12/notebook/storytelling.py +0 -684
- dartsort-0.5.12/notebook/subtract_with_phaseshift_denoiser.py +0 -260
- dartsort-0.5.12/notebook/tests/debug_matching.py +0 -454
- dartsort-0.5.12/notebook/tests/test_agreement.py +0 -367
- dartsort-0.5.12/notebook/tests/test_amplitude_scaling.py +0 -737
- dartsort-0.5.12/notebook/tests/test_batch_optim.py +0 -679
- dartsort-0.5.12/notebook/tests/test_ibme.py +0 -86
- dartsort-0.5.12/notebook/tests/test_mm_initialization.py +0 -142
- dartsort-0.5.12/notebook/tests/test_normxcorr1d.py +0 -63
- dartsort-0.5.12/notebook/tests/test_reassignment.py +0 -422
- dartsort-0.5.12/notebook/tests/test_relocation.py +0 -230
- dartsort-0.5.12/notebook/thomas.py +0 -1004
- dartsort-0.5.12/tests/test_mixture_classifier.py +0 -65
- {dartsort-0.5.12 → dartsort-0.5.13}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/.gitignore +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/README.md +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/docs/index.md +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/docs/main_api.md +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/environment.yml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/notebook/.keep +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/pyproject.toml +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/setup.cfg +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/cluster_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/kmeans.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/modes.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/clustering/refine_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/config.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/simkit.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/main.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/peel_lib.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/subtract.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/decollider.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/pipeline.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/transform_base.py +2 -2
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/main_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/motion.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/spiketorch.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/conftest.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_alignment.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_clustering.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_config.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_density.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_detect.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_linalg.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_matching.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_modeling.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_simkit.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_subtract.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_templates.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_threshold.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_transform.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.12 → dartsort-0.5.13}/zensical.toml +0 -0
|
@@ -770,7 +770,7 @@ def combine_gmm_scores(
|
|
|
770
770
|
**{
|
|
771
771
|
f"{new_prefix}_candidates": candidates,
|
|
772
772
|
f"{new_prefix}_responsibilities": responsibilities,
|
|
773
|
-
f"{new_prefix}
|
|
773
|
+
f"{new_prefix}_log_liks": logliks,
|
|
774
774
|
}
|
|
775
775
|
)
|
|
776
776
|
|
|
@@ -826,7 +826,7 @@ def combine_gmm_scores(
|
|
|
826
826
|
**{
|
|
827
827
|
f"{new_prefix}_candidates": cand,
|
|
828
828
|
f"{new_prefix}_responsibilities": mergedr,
|
|
829
|
-
f"{new_prefix}
|
|
829
|
+
f"{new_prefix}_log_liks": mergedl,
|
|
830
830
|
},
|
|
831
831
|
)
|
|
832
832
|
|
|
@@ -873,11 +873,6 @@ def deduplicate_spikes(
|
|
|
873
873
|
radius_ms: float = -1.0,
|
|
874
874
|
score_by=("merged_log_liks", "gmm_log_liks", "scores"),
|
|
875
875
|
) -> DARTsortSorting:
|
|
876
|
-
"""The lower-scoring of any spikes within radius_samples of each other is relabeled to -1.
|
|
877
|
-
|
|
878
|
-
scores are grabbed from the first entry of score_by which is a valid property of the sorting.
|
|
879
|
-
If it's mu
|
|
880
|
-
"""
|
|
881
876
|
if radius_ms < 0 or sorting.labels is None:
|
|
882
877
|
return sorting
|
|
883
878
|
|
|
@@ -155,10 +155,23 @@ class Clusterer:
|
|
|
155
155
|
def handle_sampling(
|
|
156
156
|
self, features: SimpleMatrixFeatures
|
|
157
157
|
) -> tuple[Literal[False], slice] | tuple[Literal[True], np.ndarray]:
|
|
158
|
+
if features.keep_mask is not None:
|
|
159
|
+
mask_ix = np.flatnonzero(features.keep_mask)
|
|
160
|
+
else:
|
|
161
|
+
mask_ix = None
|
|
158
162
|
if self.sampling_cfg is None:
|
|
159
|
-
|
|
163
|
+
if mask_ix is not None:
|
|
164
|
+
return True, mask_ix
|
|
165
|
+
else:
|
|
166
|
+
return False, slice(None)
|
|
160
167
|
elif features.n <= self.sampling_cfg.n_waveforms_fit:
|
|
161
|
-
|
|
168
|
+
if mask_ix is not None:
|
|
169
|
+
return True, mask_ix
|
|
170
|
+
else:
|
|
171
|
+
return False, slice(None)
|
|
172
|
+
|
|
173
|
+
if mask_ix is not None:
|
|
174
|
+
features = features.mask(mask_ix)
|
|
162
175
|
|
|
163
176
|
weights = fit_reweighting(
|
|
164
177
|
voltages=features.signed_amplitudes,
|
|
@@ -173,6 +186,8 @@ class Clusterer:
|
|
|
173
186
|
replace=False,
|
|
174
187
|
)
|
|
175
188
|
ixs.sort()
|
|
189
|
+
if mask_ix is not None:
|
|
190
|
+
ixs = mask_ix[ixs]
|
|
176
191
|
return True, ixs
|
|
177
192
|
|
|
178
193
|
def cluster(
|
|
@@ -432,16 +447,20 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
432
447
|
kdtree = res["kdtree"]
|
|
433
448
|
assert isinstance(ixs, np.ndarray)
|
|
434
449
|
rest = np.setdiff1d(np.arange(len(X)), ixs)
|
|
450
|
+
Xr = X[rest]
|
|
451
|
+
isna = np.isnan(Xr).any(axis=1)
|
|
452
|
+
np.nan_to_num(Xr, copy=False)
|
|
435
453
|
other_labels = density.nearest_neighbor_assign(
|
|
436
454
|
kdtree,
|
|
437
455
|
res["labels"],
|
|
438
|
-
|
|
456
|
+
Xr,
|
|
439
457
|
radius_search=self.radius_search,
|
|
440
458
|
workers=self.workers,
|
|
441
459
|
)
|
|
442
460
|
labels = cluster_util.combine_disjoint(
|
|
443
461
|
ixs, res["labels"], rest, other_labels
|
|
444
462
|
)
|
|
463
|
+
labels[rest[isna]] = -1
|
|
445
464
|
else:
|
|
446
465
|
kdtree = None
|
|
447
466
|
labels = res["labels"]
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from typing import Self, cast
|
|
2
|
+
from logging import WARN
|
|
2
3
|
|
|
3
4
|
import h5py
|
|
4
5
|
import numpy as np
|
|
@@ -38,9 +39,14 @@ class SimpleMatrixFeatures:
|
|
|
38
39
|
xyza: np.ndarray | None
|
|
39
40
|
signed_amplitudes: np.ndarray
|
|
40
41
|
amplitudes: np.ndarray
|
|
42
|
+
keep_mask: np.ndarray | None
|
|
41
43
|
|
|
42
44
|
def mask(self, mask) -> Self:
|
|
43
45
|
a = self.amplitudes[mask]
|
|
46
|
+
if self.keep_mask is not None:
|
|
47
|
+
k = self.keep_mask[mask]
|
|
48
|
+
else:
|
|
49
|
+
k = None
|
|
44
50
|
return self.__class__(
|
|
45
51
|
n=a.shape[0],
|
|
46
52
|
features=self.features[mask],
|
|
@@ -50,6 +56,7 @@ class SimpleMatrixFeatures:
|
|
|
50
56
|
xyza=None if self.xyza is None else self.xyza[mask],
|
|
51
57
|
signed_amplitudes=self.signed_amplitudes[mask],
|
|
52
58
|
amplitudes=a,
|
|
59
|
+
keep_mask=k,
|
|
53
60
|
)
|
|
54
61
|
|
|
55
62
|
@classmethod
|
|
@@ -68,14 +75,19 @@ class SimpleMatrixFeatures:
|
|
|
68
75
|
)
|
|
69
76
|
if xyza is not None:
|
|
70
77
|
x = xyza[:, 0]
|
|
71
|
-
|
|
72
|
-
|
|
78
|
+
_check_numbers(
|
|
79
|
+
"x", x, raise_for_numerics=clustering_features_cfg.raise_for_numerics
|
|
80
|
+
)
|
|
73
81
|
z = xyza[:, 2]
|
|
74
|
-
|
|
75
|
-
|
|
82
|
+
_check_numbers(
|
|
83
|
+
"z", z, raise_for_numerics=clustering_features_cfg.raise_for_numerics
|
|
84
|
+
)
|
|
76
85
|
z_reg = motion.correct_s(t_s, z)
|
|
77
|
-
|
|
78
|
-
|
|
86
|
+
_check_numbers(
|
|
87
|
+
"z_reg",
|
|
88
|
+
z_reg,
|
|
89
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
90
|
+
)
|
|
79
91
|
else:
|
|
80
92
|
x = z = z_reg = None
|
|
81
93
|
|
|
@@ -96,14 +108,20 @@ class SimpleMatrixFeatures:
|
|
|
96
108
|
amp = getattr(sorting, clustering_features_cfg.amplitudes_dataset_name)
|
|
97
109
|
if clustering_features_cfg.use_amplitude:
|
|
98
110
|
assert amp is not None
|
|
99
|
-
|
|
100
|
-
|
|
111
|
+
_check_numbers(
|
|
112
|
+
"amp",
|
|
113
|
+
amp,
|
|
114
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
115
|
+
)
|
|
101
116
|
ampft = amp.copy()
|
|
102
117
|
if clustering_features_cfg.log_transform_amplitude:
|
|
103
118
|
ampft = np.log(clustering_features_cfg.amp_log_c + ampft)
|
|
104
119
|
ampft *= clustering_features_cfg.amp_scale
|
|
105
|
-
|
|
106
|
-
|
|
120
|
+
_check_numbers(
|
|
121
|
+
"ampft",
|
|
122
|
+
ampft,
|
|
123
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
124
|
+
)
|
|
107
125
|
features.append(ampft[:, None])
|
|
108
126
|
|
|
109
127
|
v = getattr(sorting, clustering_features_cfg.voltages_dataset_name, None)
|
|
@@ -124,8 +142,11 @@ class SimpleMatrixFeatures:
|
|
|
124
142
|
rank=clustering_features_cfg.n_main_channel_pcs,
|
|
125
143
|
dataset_name=clustering_features_cfg.pca_dataset_name,
|
|
126
144
|
)
|
|
127
|
-
|
|
128
|
-
|
|
145
|
+
_check_numbers(
|
|
146
|
+
"No motion pcs",
|
|
147
|
+
pcs,
|
|
148
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
149
|
+
)
|
|
129
150
|
elif do_pcs and clustering_features_cfg.motion_aware:
|
|
130
151
|
shifts, n_pitches_shift = motion.pitch_shifts(
|
|
131
152
|
sorting=sorting,
|
|
@@ -144,8 +165,11 @@ class SimpleMatrixFeatures:
|
|
|
144
165
|
mask = np.broadcast_to(mask, len(schan))
|
|
145
166
|
if hasattr(sorting, clustering_features_cfg.pca_dataset_name):
|
|
146
167
|
pcs = getattr(sorting, clustering_features_cfg.pca_dataset_name)
|
|
147
|
-
|
|
148
|
-
|
|
168
|
+
_check_numbers(
|
|
169
|
+
"sorting pcs",
|
|
170
|
+
pcs,
|
|
171
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
172
|
+
)
|
|
149
173
|
erp, pcs = interpolate_by_chunk(
|
|
150
174
|
mask=mask,
|
|
151
175
|
dataset=pcs,
|
|
@@ -158,8 +182,11 @@ class SimpleMatrixFeatures:
|
|
|
158
182
|
params=clustering_features_cfg.interp_params,
|
|
159
183
|
)
|
|
160
184
|
pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
|
|
161
|
-
|
|
162
|
-
|
|
185
|
+
_check_numbers(
|
|
186
|
+
"sorting interp pcs",
|
|
187
|
+
pcs,
|
|
188
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
189
|
+
)
|
|
163
190
|
else:
|
|
164
191
|
assert sorting.parent_h5_path is not None
|
|
165
192
|
with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
|
|
@@ -175,8 +202,11 @@ class SimpleMatrixFeatures:
|
|
|
175
202
|
params=clustering_features_cfg.interp_params,
|
|
176
203
|
)
|
|
177
204
|
pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
|
|
178
|
-
|
|
179
|
-
|
|
205
|
+
_check_numbers(
|
|
206
|
+
"h5 interp pcs",
|
|
207
|
+
pcs,
|
|
208
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
209
|
+
)
|
|
180
210
|
|
|
181
211
|
if do_pcs:
|
|
182
212
|
assert pcs is not None
|
|
@@ -192,8 +222,11 @@ class SimpleMatrixFeatures:
|
|
|
192
222
|
else:
|
|
193
223
|
assert pctf in ("none", None)
|
|
194
224
|
pcs *= clustering_features_cfg.pc_scale
|
|
195
|
-
|
|
196
|
-
|
|
225
|
+
_check_numbers(
|
|
226
|
+
f"{pctf} pcs",
|
|
227
|
+
pcs,
|
|
228
|
+
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
229
|
+
)
|
|
197
230
|
if torch.is_tensor(pcs):
|
|
198
231
|
pcs = pcs.numpy(force=True)
|
|
199
232
|
features.append(pcs)
|
|
@@ -203,6 +236,9 @@ class SimpleMatrixFeatures:
|
|
|
203
236
|
features = np.concatenate(features, axis=1)
|
|
204
237
|
else:
|
|
205
238
|
features = np.empty((n, 0))
|
|
239
|
+
keep = np.atleast_1d(np.isfinite(features).all(axis=1))
|
|
240
|
+
if keep.all():
|
|
241
|
+
keep = None
|
|
206
242
|
return cls(
|
|
207
243
|
n=n,
|
|
208
244
|
features=features,
|
|
@@ -212,6 +248,7 @@ class SimpleMatrixFeatures:
|
|
|
212
248
|
xyza=xyza,
|
|
213
249
|
amplitudes=amp,
|
|
214
250
|
signed_amplitudes=samp,
|
|
251
|
+
keep_mask=keep,
|
|
215
252
|
)
|
|
216
253
|
|
|
217
254
|
|
|
@@ -331,7 +368,18 @@ def _allfinite(x):
|
|
|
331
368
|
return np.isfinite(x).all()
|
|
332
369
|
|
|
333
370
|
|
|
334
|
-
def
|
|
371
|
+
def _check_numbers(name: str, x: np.ndarray, raise_for_numerics=False):
|
|
335
372
|
if isinstance(x, torch.Tensor):
|
|
336
373
|
x = x.numpy(force=True)
|
|
337
|
-
|
|
374
|
+
|
|
375
|
+
npinf = np.isposinf(x).sum()
|
|
376
|
+
nninf = np.isneginf(x).sum()
|
|
377
|
+
nna = np.isnan(x).sum()
|
|
378
|
+
if not (npinf + nninf + nna):
|
|
379
|
+
return
|
|
380
|
+
|
|
381
|
+
err = f"{name}: {npinf} +inf, {nninf} -inf, {nna} nan."
|
|
382
|
+
if raise_for_numerics:
|
|
383
|
+
raise ValueError(err)
|
|
384
|
+
else:
|
|
385
|
+
logger.warn(err)
|
|
@@ -4381,7 +4381,7 @@ def get_full_neighborhood_data(
|
|
|
4381
4381
|
assert xfull.shape[2] == full_neighborhoods.b.neighborhoods.shape[1]
|
|
4382
4382
|
frank = xfull.shape[1]
|
|
4383
4383
|
if clustering_features_cfg is not None:
|
|
4384
|
-
assert frank
|
|
4384
|
+
assert 0 < frank <= clustering_features_cfg.feature_rank
|
|
4385
4385
|
assert xfull.shape[0] == full_neighborhoods.b.neighborhood_ids.shape[0]
|
|
4386
4386
|
xfull = xfull.view(len(xfull), -1)
|
|
4387
4387
|
xfull = xfull.nan_to_num_()
|
|
@@ -108,6 +108,8 @@ class DARTsortAnalysis:
|
|
|
108
108
|
"localizations",
|
|
109
109
|
"positions",
|
|
110
110
|
),
|
|
111
|
+
compute_extensions=("correlograms",),
|
|
112
|
+
estimate_si_sparsity=False,
|
|
111
113
|
try_backup_amplitudes=("amplitudes", "ptp_amplitudes"),
|
|
112
114
|
quality_metrics=(
|
|
113
115
|
"sliding_rp_violation",
|
|
@@ -288,6 +290,8 @@ class DARTsortAnalysis:
|
|
|
288
290
|
motion=motion,
|
|
289
291
|
features_cfg=clustering_features_cfg,
|
|
290
292
|
compute_extensions_if_templates=None,
|
|
293
|
+
compute_extensions=compute_extensions,
|
|
294
|
+
estimate_si_sparsity=estimate_si_sparsity,
|
|
291
295
|
)
|
|
292
296
|
analyzer.compute("quality_metrics", metric_names=quality_metrics)
|
|
293
297
|
qc_df = analyzer.get_extension("quality_metrics").get_data()
|
|
@@ -379,7 +383,10 @@ class DARTsortAnalysis:
|
|
|
379
383
|
return self._summary_df
|
|
380
384
|
df = self.quality_df.copy()
|
|
381
385
|
uids = self.sorting.unit_ids
|
|
382
|
-
|
|
386
|
+
if self.template_data is not None:
|
|
387
|
+
df["ds_unit_amplitudes"] = pd.Series(
|
|
388
|
+
index=uids, data=self.unit_amplitudes()
|
|
389
|
+
)
|
|
383
390
|
df["ds_firing_rates"] = pd.Series(index=uids, data=self.firing_rates())
|
|
384
391
|
self._summary_df = df
|
|
385
392
|
return df
|
|
@@ -135,12 +135,12 @@ class DARTsortGroundTruthComparison:
|
|
|
135
135
|
return res
|
|
136
136
|
|
|
137
137
|
def _unit_info_dataframe(self, force_distances=False, perf_only=False):
|
|
138
|
-
amplitudes = self.gt_analysis.unit_amplitudes()
|
|
139
138
|
firing_rates = self.gt_analysis.firing_rates()
|
|
140
139
|
df = self.comparison.get_performance()
|
|
141
140
|
assert isinstance(df, pd.DataFrame)
|
|
142
141
|
df = df.astype(float) # not sure what the problem was...
|
|
143
|
-
|
|
142
|
+
if self.gt_analysis.template_data is not None:
|
|
143
|
+
df["gt_ptp_amplitude"] = self.gt_analysis.unit_amplitudes()
|
|
144
144
|
df["gt_firing_rate"] = firing_rates
|
|
145
145
|
df["gt_sorter_name"] = self.gt_analysis.name or ""
|
|
146
146
|
df["tested_sorter_name"] = self.tested_analysis.name or ""
|
|
@@ -88,8 +88,10 @@ class GrabAndFeaturize(BasePeeler):
|
|
|
88
88
|
skip_features: bool = False,
|
|
89
89
|
n_resid_snips: int | None = None,
|
|
90
90
|
to_cpu: bool = True,
|
|
91
|
+
**peel_kw,
|
|
91
92
|
) -> "PeelingBatchResult":
|
|
92
93
|
"""Override process_chunk to skip empties."""
|
|
94
|
+
assert not peel_kw
|
|
93
95
|
if chunk_end_samples is None:
|
|
94
96
|
chunk_end_samples = min(
|
|
95
97
|
self.recording.get_num_samples(),
|
|
@@ -24,7 +24,7 @@ from ..util.internal_config import (
|
|
|
24
24
|
)
|
|
25
25
|
from ..util.logging_util import get_logger
|
|
26
26
|
from ..util.motion import MotionInfo
|
|
27
|
-
from ..util.waveform_util import make_channel_index
|
|
27
|
+
from ..util.waveform_util import full_channel_index, make_channel_index
|
|
28
28
|
from .matching_util import (
|
|
29
29
|
ChunkTemplateData,
|
|
30
30
|
MatchingPeaks,
|
|
@@ -260,6 +260,7 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
|
|
|
260
260
|
return_residual=False,
|
|
261
261
|
return_waveforms=True,
|
|
262
262
|
return_conv=False,
|
|
263
|
+
return_clean_waveforms=False,
|
|
263
264
|
) -> PeelingBatchResult:
|
|
264
265
|
assert self.matching_templates is not None
|
|
265
266
|
# get chunk center time and template info at that time
|
|
@@ -288,6 +289,7 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
|
|
|
288
289
|
return_residual=return_residual,
|
|
289
290
|
return_conv=return_conv,
|
|
290
291
|
return_collisioncleaned_waveforms=return_waveforms,
|
|
292
|
+
return_clean_waveforms=return_clean_waveforms,
|
|
291
293
|
)
|
|
292
294
|
|
|
293
295
|
# process spike times and create return result
|
|
@@ -304,9 +306,11 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
|
|
|
304
306
|
self,
|
|
305
307
|
traces: Tensor,
|
|
306
308
|
chunk_template_data: ChunkTemplateData,
|
|
309
|
+
*,
|
|
307
310
|
left_margin=0,
|
|
308
311
|
right_margin=0,
|
|
309
312
|
return_collisioncleaned_waveforms=True,
|
|
313
|
+
return_clean_waveforms=False,
|
|
310
314
|
return_residual=False,
|
|
311
315
|
return_conv=False,
|
|
312
316
|
unit_mask=None,
|
|
@@ -475,6 +479,14 @@ class ObjectiveUpdateTemplateMatchingPeeler(BasePeeler):
|
|
|
475
479
|
assert self.p.channel_selection == "template"
|
|
476
480
|
channels = chunk_template_data.main_channels[peaks.template_inds]
|
|
477
481
|
waveforms = None
|
|
482
|
+
if return_clean_waveforms:
|
|
483
|
+
ci_full = full_channel_index(self.b.channel_index.shape[0], True)
|
|
484
|
+
ci_full = ci_full.to(self.b.channel_index)
|
|
485
|
+
res["clean_waveforms"] = chunk_template_data.get_clean_waveforms(
|
|
486
|
+
peaks=peaks,
|
|
487
|
+
channels=channels,
|
|
488
|
+
channel_index=ci_full,
|
|
489
|
+
)
|
|
478
490
|
res["channels"] = channels
|
|
479
491
|
if return_collisioncleaned_waveforms:
|
|
480
492
|
assert waveforms is not None
|
|
@@ -496,6 +496,7 @@ class BasePeeler(BModule):
|
|
|
496
496
|
skip_features: bool = False,
|
|
497
497
|
n_resid_snips: int | None = None,
|
|
498
498
|
to_cpu: bool = True,
|
|
499
|
+
**peel_kw,
|
|
499
500
|
) -> "PeelingBatchResult":
|
|
500
501
|
"""Grab, peel, and featurize a chunk, returning a dict of numpy arrays
|
|
501
502
|
|
|
@@ -512,6 +513,7 @@ class BasePeeler(BModule):
|
|
|
512
513
|
right_margin=right_margin,
|
|
513
514
|
return_waveforms=return_waveforms,
|
|
514
515
|
return_residual=return_residual or bool(n_resid_snips),
|
|
516
|
+
**peel_kw,
|
|
515
517
|
)
|
|
516
518
|
chunk_result = self.featurize_chunk_result(
|
|
517
519
|
peel_result=peel_result,
|
|
@@ -162,7 +162,7 @@ class AmortizedLocalization(BaseWaveformFeaturizer):
|
|
|
162
162
|
assert obs_amps is not None
|
|
163
163
|
if neighborhoods is None:
|
|
164
164
|
neighborhoods = self.b.padded_geom[self.b.model_channel_index[channels]]
|
|
165
|
-
w = obs_amps / obs_amps.sum(1, keepdims=True)
|
|
165
|
+
w = obs_amps / obs_amps.sum(1, keepdims=True).clamp(min=1e-6)
|
|
166
166
|
centers = torch.sum(w.unsqueeze(-1) * neighborhoods, dim=1)
|
|
167
167
|
return centers
|
|
168
168
|
else:
|
|
@@ -266,9 +266,9 @@ class AmortizedLocalization(BaseWaveformFeaturizer):
|
|
|
266
266
|
x_masked = x * mask
|
|
267
267
|
if self.scale_loss_by_mean:
|
|
268
268
|
# 1/(n_chans_retained*mean amplitude)
|
|
269
|
-
rescale = x_masked.sum(1, keepdim=True).clamp(min=1e-
|
|
269
|
+
rescale = x_masked.sum(1, keepdim=True).clamp(min=1e-6).reciprocal()
|
|
270
270
|
else:
|
|
271
|
-
rescale =
|
|
271
|
+
rescale = mask.sum(1, keepdim=True).clamp(min=1e-6).reciprocal()
|
|
272
272
|
x_masked *= rescale
|
|
273
273
|
recon_x_masked *= rescale
|
|
274
274
|
mse = F.mse_loss(recon_x_masked, x_masked, reduction="sum") / self.batch_size
|
|
@@ -13,6 +13,7 @@ TODO:
|
|
|
13
13
|
|
|
14
14
|
import sys
|
|
15
15
|
from dataclasses import replace
|
|
16
|
+
from math import fabs
|
|
16
17
|
from typing import TYPE_CHECKING, Sequence
|
|
17
18
|
|
|
18
19
|
import numpy as np
|
|
@@ -113,6 +114,11 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
|
|
|
113
114
|
self.channel_index_np = self.b.channel_index.numpy(force=True)
|
|
114
115
|
self.workers = 1
|
|
115
116
|
|
|
117
|
+
def get_extra_state(self):
|
|
118
|
+
es = super().get_extra_state()
|
|
119
|
+
es["feature_rank"] = self.feature_rank
|
|
120
|
+
return es
|
|
121
|
+
|
|
116
122
|
def needs_fit(self):
|
|
117
123
|
return not hasattr(self, "tmm")
|
|
118
124
|
|
|
@@ -146,6 +152,48 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
|
|
|
146
152
|
cii, cjj = np.nonzero(self.channel_index_np < len(self.channel_index_np))
|
|
147
153
|
self.channel_index_valid_inds = cii, cjj
|
|
148
154
|
|
|
155
|
+
# precompute drifting neighborhood lookup tables
|
|
156
|
+
if self.motion.drifting:
|
|
157
|
+
# in this case, prebake a lookup table
|
|
158
|
+
# TODO just consider relevant time bins here?
|
|
159
|
+
# or chunk this somehow?
|
|
160
|
+
tbs = self.motion.time_bins_s
|
|
161
|
+
tcm_t, nid_t = neighborhood_mapping_at_time(
|
|
162
|
+
motion=self.motion,
|
|
163
|
+
t_s=torch.from_numpy(tbs),
|
|
164
|
+
neighborhoods=self.b.neighborhoods,
|
|
165
|
+
channel_index=self.channel_index_np,
|
|
166
|
+
workers=self.workers,
|
|
167
|
+
static_neighbs=self.static_neighbs,
|
|
168
|
+
channel_index_valid_inds=self.channel_index_valid_inds,
|
|
169
|
+
)
|
|
170
|
+
nid_map = None
|
|
171
|
+
else:
|
|
172
|
+
tcm_t = nid_t = tbs = None
|
|
173
|
+
# in this case, channel index is a superset of neighborhoods and
|
|
174
|
+
# we prebake the mapping from chans to neighb ids
|
|
175
|
+
ci_eq_neighb = _outer_all_equal(
|
|
176
|
+
self.b.channel_index, self.tmm.neighb_cov.obs_ix
|
|
177
|
+
)
|
|
178
|
+
# some neighborhood entries are identical, but they all get covered
|
|
179
|
+
assert torch.all(ci_eq_neighb.sum(0) >= 1)
|
|
180
|
+
# each channel index maps to at most one neighb id
|
|
181
|
+
assert torch.all(ci_eq_neighb.sum(1) <= 1)
|
|
182
|
+
_chans, _chan_nids = ci_eq_neighb.nonzero(as_tuple=True)
|
|
183
|
+
nid_map = _chans.new_full(
|
|
184
|
+
(self.b.channel_index.shape[0],),
|
|
185
|
+
self.tmm.neighb_cov.b.obs_ix.shape[0] + 1,
|
|
186
|
+
)
|
|
187
|
+
nid_map[_chans] = _chan_nids
|
|
188
|
+
self.register_buffer_or_none("neighborhood_ids_map", nid_map)
|
|
189
|
+
self.drifting_time_bins_s = tbs
|
|
190
|
+
self.register_buffer_or_none(
|
|
191
|
+
"drifting_target_channels_map_t", tcm_t, persistent=False
|
|
192
|
+
)
|
|
193
|
+
self.register_buffer_or_none(
|
|
194
|
+
"drifting_neighborhood_ids_map_t", nid_t, persistent=False
|
|
195
|
+
)
|
|
196
|
+
|
|
149
197
|
def fit(
|
|
150
198
|
self,
|
|
151
199
|
recording,
|
|
@@ -211,24 +259,6 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
|
|
|
211
259
|
assert mix_data.tmm.noise is not None
|
|
212
260
|
self.feature_rank = mix_data.tmm.noise.rank
|
|
213
261
|
|
|
214
|
-
if not self.motion.drifting:
|
|
215
|
-
# in this case, channel index is a superset of neighborhoods and
|
|
216
|
-
# we prebake the mapping from chans to neighb ids
|
|
217
|
-
ci_eq_neighb = _outer_all_equal(
|
|
218
|
-
self.b.channel_index, mix_data.tmm.neighb_cov.obs_ix
|
|
219
|
-
)
|
|
220
|
-
# some neighborhood entries are identical, but they all get covered
|
|
221
|
-
assert torch.all(ci_eq_neighb.sum(0) >= 1)
|
|
222
|
-
# each channel index maps to at most one neighb id
|
|
223
|
-
assert torch.all(ci_eq_neighb.sum(1) <= 1)
|
|
224
|
-
_chans, _chan_nids = ci_eq_neighb.nonzero(as_tuple=True)
|
|
225
|
-
nid_map = _chans.new_full(
|
|
226
|
-
(self.b.channel_index.shape[0],),
|
|
227
|
-
mix_data.tmm.neighb_cov.b.obs_ix.shape[0] + 1,
|
|
228
|
-
)
|
|
229
|
-
nid_map[_chans] = _chan_nids
|
|
230
|
-
self.register_buffer("neighborhood_ids_map", nid_map)
|
|
231
|
-
|
|
232
262
|
self.erp: StableFeaturesInterpolator = stable_features.erp
|
|
233
263
|
|
|
234
264
|
# de-puff the lut for space reasons
|
|
@@ -268,6 +298,8 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
|
|
|
268
298
|
|
|
269
299
|
assert hasattr(self, "motion")
|
|
270
300
|
assert self.motion is not None
|
|
301
|
+
extra_state = state_dict[f"{prefix}_extra_state"]
|
|
302
|
+
self.feature_rank = extra_state["feature_rank"]
|
|
271
303
|
|
|
272
304
|
# reconstruct non-__init__ modules...
|
|
273
305
|
stripped_dict = {k.removeprefix(prefix): v for k, v in state_dict.items()}
|
|
@@ -318,15 +350,10 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
|
|
|
318
350
|
# drifting channel mapping
|
|
319
351
|
assert self.motion is not None
|
|
320
352
|
if self.motion.drifting:
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
channel_index=self.channel_index_np,
|
|
326
|
-
workers=self.workers,
|
|
327
|
-
static_neighbs=self.static_neighbs,
|
|
328
|
-
channel_index_valid_inds=self.channel_index_valid_inds,
|
|
329
|
-
)
|
|
353
|
+
assert self.drifting_time_bins_s is not None
|
|
354
|
+
t_idx = find_nearest(self.drifting_time_bins_s, chunk_center_s)
|
|
355
|
+
target_channels_map = self.b.drifting_target_channels_map_t[t_idx]
|
|
356
|
+
neighborhood_ids_map = self.b.drifting_neighborhood_ids_map_t[t_idx]
|
|
330
357
|
else:
|
|
331
358
|
target_channels_map = self.b.channel_index
|
|
332
359
|
neighborhood_ids_map = self.b.neighborhood_ids_map
|
|
@@ -376,62 +403,57 @@ class TruncatedMixtureModelTransformer(BaseWaveformFeaturizer):
|
|
|
376
403
|
|
|
377
404
|
def neighborhood_mapping_at_time(
|
|
378
405
|
motion: MotionInfo,
|
|
379
|
-
|
|
406
|
+
*,
|
|
407
|
+
t_s: torch.Tensor | np.ndarray,
|
|
380
408
|
channel_index: np.ndarray,
|
|
381
409
|
neighborhoods: torch.Tensor,
|
|
382
410
|
workers: int = 4,
|
|
383
411
|
shift_mode="round",
|
|
384
|
-
static_neighbs: np.ndarray
|
|
385
|
-
channel_index_valid_inds: tuple[np.ndarray, np.ndarray]
|
|
412
|
+
static_neighbs: np.ndarray,
|
|
413
|
+
channel_index_valid_inds: tuple[np.ndarray, np.ndarray],
|
|
386
414
|
):
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
t_s
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
n_pitches_shift = (probe_disp / motion.pitch).astype(np.int32)
|
|
397
|
-
elif shift_mode == "round":
|
|
398
|
-
n_pitches_shift = np.round(probe_disp / motion.pitch).astype(np.int32)
|
|
399
|
-
else:
|
|
400
|
-
assert False
|
|
401
|
-
|
|
402
|
-
if channel_index_valid_inds is None:
|
|
403
|
-
cii, cjj = np.nonzero(channel_index < len(channel_index))
|
|
404
|
-
else:
|
|
405
|
-
cii, cjj = channel_index_valid_inds
|
|
415
|
+
t_s = t_s.numpy(force=True) if isinstance(t_s, torch.Tensor) else np.asarray(t_s)
|
|
416
|
+
t_s = np.atleast_1d(t_s)
|
|
417
|
+
|
|
418
|
+
index_at_time = torch.full((len(t_s), *channel_index.shape), -1)
|
|
419
|
+
nids_at_time = torch.full(
|
|
420
|
+
(len(t_s), channel_index.shape[0]),
|
|
421
|
+
neighborhoods.shape[0],
|
|
422
|
+
device=neighborhoods.device,
|
|
423
|
+
)
|
|
406
424
|
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
425
|
+
for j, t in enumerate(t_s):
|
|
426
|
+
probe_disp = -motion.disp_at_s(
|
|
427
|
+
times_s=t, depths_um=motion.geom[:, 1], grid=True
|
|
428
|
+
)
|
|
429
|
+
if shift_mode == "floor":
|
|
430
|
+
n_pitches_shift = (probe_disp / motion.pitch).astype(np.int32)
|
|
431
|
+
elif shift_mode == "round":
|
|
432
|
+
n_pitches_shift = np.round(probe_disp / motion.pitch).astype(np.int32)
|
|
433
|
+
else:
|
|
434
|
+
assert False
|
|
435
|
+
shift = n_pitches_shift * motion.pitch
|
|
412
436
|
shifted_neighbs = static_neighbs.copy()
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
437
|
+
shifted_neighbs[:, :, 1] += shift
|
|
438
|
+
_, umatch = motion.rgeom_kdt.query(
|
|
439
|
+
shifted_neighbs[channel_index_valid_inds],
|
|
440
|
+
distance_upper_bound=motion.min_dist,
|
|
441
|
+
workers=workers,
|
|
442
|
+
)
|
|
443
|
+
index = np.full(channel_index.shape, motion.rgeom.shape[0])
|
|
444
|
+
index[channel_index_valid_inds] = umatch
|
|
445
|
+
index = torch.asarray(index, device=neighborhoods.device)
|
|
446
|
+
assert index.shape[1] == neighborhoods.shape[1]
|
|
416
447
|
|
|
417
|
-
|
|
418
|
-
_, umatch = motion.rgeom_kdt.query(
|
|
419
|
-
shifted_neighbs[cii, cjj], distance_upper_bound=motion.min_dist, workers=workers
|
|
420
|
-
)
|
|
448
|
+
index_at_time[j] = index
|
|
421
449
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
450
|
+
mapping = _outer_all_equal(index, neighborhoods)
|
|
451
|
+
_chans, _chan_nids = mapping.nonzero(as_tuple=True)
|
|
452
|
+
nids_at_time[j, _chans] = _chan_nids
|
|
425
453
|
|
|
426
|
-
|
|
427
|
-
index = torch.asarray(index, device=neighborhoods.device)
|
|
428
|
-
assert index.shape[1] == neighborhoods.shape[1]
|
|
429
|
-
mapping = _outer_all_equal(index, neighborhoods)
|
|
430
|
-
_chans, _chan_nids = mapping.nonzero(as_tuple=True)
|
|
431
|
-
nid_map = _chans.new_full((channel_index.shape[0],), neighborhoods.shape[0])
|
|
432
|
-
nid_map[_chans] = _chan_nids
|
|
454
|
+
assert (index_at_time >= 0).all()
|
|
433
455
|
|
|
434
|
-
return
|
|
456
|
+
return index_at_time, nids_at_time
|
|
435
457
|
|
|
436
458
|
|
|
437
459
|
@torch_compile
|
|
@@ -451,3 +473,11 @@ def _outer_all_equal(x: torch.Tensor, y: torch.Tensor):
|
|
|
451
473
|
out.logical_and_(torch.eq(x[:, :, j], y[:, :, j], out=msk))
|
|
452
474
|
|
|
453
475
|
return out
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def find_nearest(x, v):
|
|
479
|
+
idx = np.searchsorted(x, v, side="left")
|
|
480
|
+
if idx > 0 and (idx == len(x) or fabs(v - x[idx - 1]) < fabs(v - x[idx])):
|
|
481
|
+
return idx - 1
|
|
482
|
+
else:
|
|
483
|
+
return idx
|