DARTsort 0.5.3__tar.gz → 0.5.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.3 → dartsort-0.5.5}/PKG-INFO +1 -1
- {dartsort-0.5.3 → dartsort-0.5.5}/docs/main_api.md +3 -2
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/__init__.py +2 -2
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/kmeans.py +2 -1
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/mixture.py +43 -43
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/config.py +1 -2
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/hybrid_util.py +7 -7
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/simkit.py +11 -7
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/main.py +22 -15
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/pairwise.py +2 -2
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/reduction_template.py +8 -3
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/postprocess_util.py +2 -2
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/amortized_localization.py +3 -3
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/pipeline.py +3 -3
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/reduction.py +1 -1
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/single_channel_denoiser.py +2 -2
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/cli_util.py +0 -1
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/data_util.py +21 -15
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/internal_config.py +4 -2
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/logging_util.py +25 -18
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/main_util.py +14 -9
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/motion.py +18 -9
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/multiprocessing_util.py +9 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/noise_util.py +4 -3
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/peel_util.py +2 -2
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/preprocess_util.py +4 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/py_util.py +17 -6
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/spiketorch.py +16 -6
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/waveform_util.py +1 -1
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/colors.py +2 -2
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/mixture.py +100 -37
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/scatterplots.py +4 -1
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/unit.py +1 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/conftest.py +6 -6
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_alignment.py +1 -1
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_matching.py +1 -1
- {dartsort-0.5.3 → dartsort-0.5.5}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/.gitignore +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/README.md +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/docs/index.md +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/environment.yml +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/.keep +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/basic_integration_test.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/ci diag.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/dartsort_dredge_demo.ipynb +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/dartsort_dredge_demo.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/data test.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/dead reckoning.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/denoise np1.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/denoise np2.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/exploratory.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/geom debug.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/hdbscan_clustering_5min.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/hybrid 10.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/hybrid cluster.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/hybrid_11.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/initial training.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/make hybrid.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/mgh.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/np1 feats.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/np2 processing version c.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/paperfigs.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/pdists.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/postcheck.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/ptp vae.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/re clust.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/reg fig loop.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/reg_sim_exp.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/relocating.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/relocating_all.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/restats +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/sortStinger1.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/sortStinger3.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/sortStinger4.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/sortStinger5.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/sortStinger6.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/spike farm.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/ss_ensemble_sim.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/ss_ensemble_sim_drift.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/ss_sim.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/stan diagnostics.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/storytelling c.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/storytelling.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/subtract_with_phaseshift_denoiser.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/debug_matching.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_agreement.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_amplitude_scaling.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_batch_optim.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_ibme.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_mm_initialization.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_normxcorr1d.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_reassignment.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/tests/test_relocation.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/notebook/thomas.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/pyproject.toml +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/setup.cfg +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/agglomerate.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/cluster_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/clustering.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/clustering_features.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/modes.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/clustering/refine_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/analysis.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/comparison.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/peel_base.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/peel_lib.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/subtract.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/decollider.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/mixture_classifier.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/transform_base.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/analysis_plots.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/vismain.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/SOURCES.txt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_clustering.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_config.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_data_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_detect.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_linalg.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_modeling.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_simkit.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_subtract.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_templates.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_threshold.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_transform.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.3 → dartsort-0.5.5}/zensical.toml +0 -0
|
@@ -4,7 +4,8 @@ toc_depth: 3
|
|
|
4
4
|
|
|
5
5
|
# Main API: `dartsort()` and the `DARTsortUserConfig`
|
|
6
6
|
|
|
7
|
-
This page shows the main functions and objects that you'd run into when spike sorting with *dartsort*.
|
|
7
|
+
This page shows the main functions and objects that you'd run into when spike sorting with *dartsort*.
|
|
8
|
+
If you're just getting started, the [usage section on the front page](/#usage) might be good to read first.
|
|
8
9
|
|
|
9
10
|
For details on parameters you should think about before running the sorter, see the [important configuration details note](/#important-configuration-details).
|
|
10
11
|
|
|
@@ -16,7 +17,7 @@ For details on parameters you should think about before running the sorter, see
|
|
|
16
17
|
show_signature: true
|
|
17
18
|
separate_signature: true
|
|
18
19
|
|
|
19
|
-
The return value from the `dartsort()` function is a
|
|
20
|
+
The return value from the `dartsort()` function is a `DARTsortResult` object, which is a dictionary containing spike trains and motion information:
|
|
20
21
|
|
|
21
22
|
::: dartsort.DARTsortReturn
|
|
22
23
|
options:
|
|
@@ -24,7 +24,7 @@ from .localize.localize_util import (
|
|
|
24
24
|
from .main import (
|
|
25
25
|
ObjectiveUpdateTemplateMatchingPeeler,
|
|
26
26
|
SubtractionPeeler,
|
|
27
|
-
|
|
27
|
+
DARTsortResult,
|
|
28
28
|
check_recording,
|
|
29
29
|
cluster,
|
|
30
30
|
dartsort,
|
|
@@ -78,7 +78,7 @@ from .util.logging_util import (
|
|
|
78
78
|
from .util.motion import MotionInfo, get_motion_info, try_load_motion_info
|
|
79
79
|
from .util.noise_util import EmbeddedNoise
|
|
80
80
|
from .util.preprocess_util import preprocess
|
|
81
|
-
from .util.py_util import databag,
|
|
81
|
+
from .util.py_util import databag, ensure_path
|
|
82
82
|
from .util.waveform_util import full_channel_index, make_channel_index
|
|
83
83
|
|
|
84
84
|
__version__ = importlib.metadata.version("dartsort")
|
|
@@ -22,7 +22,6 @@ from ..util.sparse_util import (
|
|
|
22
22
|
sparse_centroid_distsq,
|
|
23
23
|
)
|
|
24
24
|
from ..util.spiketorch import spawn_torch_rg
|
|
25
|
-
from .density import guess_mode
|
|
26
25
|
|
|
27
26
|
logger = get_logger(__name__)
|
|
28
27
|
|
|
@@ -58,6 +57,8 @@ def kmeanspp(
|
|
|
58
57
|
closest = torch.cdist(X, X.mean(0, keepdim=True)).argmax()
|
|
59
58
|
centroid_ixs[0] = closest.item()
|
|
60
59
|
elif kmeanspp_initial == "mode":
|
|
60
|
+
from .density import guess_mode
|
|
61
|
+
|
|
61
62
|
Xm = X
|
|
62
63
|
if Xm.shape[1] > mode_dim:
|
|
63
64
|
q = min(mode_dim + 10, *Xm.shape)
|
|
@@ -220,9 +220,11 @@ def tmm_demix(
|
|
|
220
220
|
)
|
|
221
221
|
else:
|
|
222
222
|
assert False
|
|
223
|
+
|
|
223
224
|
stepname = f"tmm{outer_it}{inner_it}{step_type}"
|
|
224
|
-
|
|
225
|
-
|
|
225
|
+
last_outer = outer_it == refinement_cfg.n_total_iters - 1
|
|
226
|
+
last_inner = inner_it == len(refinement_cfg.mixture_steps) - 1
|
|
227
|
+
is_final = last_outer and last_inner
|
|
226
228
|
if saving and not is_final:
|
|
227
229
|
save_tmm_labels(tmm=tmm, stepname=stepname, **save_kw) # type: ignore
|
|
228
230
|
|
|
@@ -1317,10 +1319,13 @@ class BatchedSpikeData:
|
|
|
1317
1319
|
self.candidates.fill_(-1)
|
|
1318
1320
|
|
|
1319
1321
|
def bootstrap_candidates(
|
|
1320
|
-
self,
|
|
1322
|
+
self,
|
|
1323
|
+
distances: Tensor,
|
|
1324
|
+
un_adj_lut: NeighborhoodLUT | None = None,
|
|
1325
|
+
allow_uncovered: bool = False,
|
|
1321
1326
|
) -> NeighborhoodLUT:
|
|
1322
1327
|
self.update_adjacency(n_units=distances.shape[0], un_adj_lut=un_adj_lut)
|
|
1323
|
-
self._fill_missing(distances.shape[0])
|
|
1328
|
+
self._fill_missing(distances.shape[0], allow_uncovered=allow_uncovered)
|
|
1324
1329
|
|
|
1325
1330
|
# fill in candidates[:, 1:n_candidates] at random obeying un_adj
|
|
1326
1331
|
# choosing not to use distances here, since they get used in search sets
|
|
@@ -1859,7 +1864,7 @@ class TruncatedSpikeData(BatchedSpikeData):
|
|
|
1859
1864
|
# have to do a full bootstrap, bc it's hard to figure out what to do with
|
|
1860
1865
|
# spikes whose candidates contain the units that were split. this way, the
|
|
1861
1866
|
# lut invariants are maintained, and at least the top labels are the same.
|
|
1862
|
-
return self.bootstrap_candidates(distances)
|
|
1867
|
+
return self.bootstrap_candidates(distances, allow_uncovered=True)
|
|
1863
1868
|
|
|
1864
1869
|
def full_proposal_view(self, un_adj_lut: NeighborhoodLUT):
|
|
1865
1870
|
return FullProposalDataView.from_truncated_spike_data(self, un_adj_lut)
|
|
@@ -4050,6 +4055,10 @@ def get_truncated_datasets(
|
|
|
4050
4055
|
device=device,
|
|
4051
4056
|
rgeom=prgeom[:-1],
|
|
4052
4057
|
)
|
|
4058
|
+
|
|
4059
|
+
gc.collect()
|
|
4060
|
+
torch.cuda.empty_cache()
|
|
4061
|
+
|
|
4053
4062
|
assert isinstance(noise, EmbeddedNoise)
|
|
4054
4063
|
noise.to(device=device)
|
|
4055
4064
|
assert noise.rank == feature_rank
|
|
@@ -4152,6 +4161,9 @@ def get_truncated_datasets(
|
|
|
4152
4161
|
else:
|
|
4153
4162
|
assert False
|
|
4154
4163
|
|
|
4164
|
+
gc.collect()
|
|
4165
|
+
torch.cuda.empty_cache()
|
|
4166
|
+
|
|
4155
4167
|
return neighb_cov, erp, train_data, val_data, full_data, noise, train_ixs, val_ixs
|
|
4156
4168
|
|
|
4157
4169
|
|
|
@@ -5197,7 +5209,7 @@ def try_kmeans(
|
|
|
5197
5209
|
feature_rank: int,
|
|
5198
5210
|
n_iter: int = 100,
|
|
5199
5211
|
with_proportions: bool = True,
|
|
5200
|
-
drop_prop: float = 0.
|
|
5212
|
+
drop_prop: float = 0.0,
|
|
5201
5213
|
kmeanspp_initial="random",
|
|
5202
5214
|
n_kmeans_tries: int = 25,
|
|
5203
5215
|
n_kmeanspp_tries: int = 25,
|
|
@@ -5251,10 +5263,7 @@ def evaluate_group_demolitions(
|
|
|
5251
5263
|
train_scores: Scores,
|
|
5252
5264
|
eval_scores: Scores,
|
|
5253
5265
|
) -> GroupDemolition:
|
|
5254
|
-
# grab relevant sections of train and eval scores
|
|
5255
5266
|
group_ = group.to(train_scores.candidates)
|
|
5256
|
-
in_group_train = torch.isin(train_scores.candidates, group_).any(dim=1)
|
|
5257
|
-
(in_group_train,) = in_group_train.nonzero(as_tuple=True)
|
|
5258
5267
|
|
|
5259
5268
|
# select which units in the group are candidates for demolition
|
|
5260
5269
|
if mean_train_resp is None:
|
|
@@ -6059,36 +6068,30 @@ def _count_candidates(candidates, batch_candidate_counts, batch_size):
|
|
|
6059
6068
|
|
|
6060
6069
|
if TORCH_IS_OLD:
|
|
6061
6070
|
|
|
6062
|
-
|
|
6063
|
-
|
|
6064
|
-
|
|
6065
|
-
|
|
6066
|
-
discard_mask = torch.logical_not(keep_mask)
|
|
6067
|
-
discard_ix = discard_mask.nonzero()[:, 0]
|
|
6068
|
-
keep_ix = keep_mask.nonzero()[:, 0]
|
|
6069
|
-
assert keep_ix.numel() + discard_ix.numel() == resps.shape[1]
|
|
6070
|
-
kept_resp = resps[:, keep_ix]
|
|
6071
|
-
discard_resp = resps[:, discard_ix]
|
|
6072
|
-
sim = discard_resp.T @ kept_resp
|
|
6073
|
-
match = sim.argmax(1)
|
|
6074
|
-
kept_resp[:, match] += discard_resp
|
|
6075
|
-
return kept_resp
|
|
6071
|
+
def _nonzero_static(x: Tensor, size: int):
|
|
6072
|
+
nz = x.nonzero()
|
|
6073
|
+
assert nz.numel() == size
|
|
6074
|
+
return nz
|
|
6076
6075
|
else:
|
|
6077
6076
|
|
|
6078
|
-
|
|
6079
|
-
|
|
6080
|
-
|
|
6081
|
-
|
|
6082
|
-
|
|
6083
|
-
|
|
6084
|
-
|
|
6085
|
-
|
|
6086
|
-
|
|
6087
|
-
|
|
6088
|
-
|
|
6089
|
-
|
|
6090
|
-
|
|
6091
|
-
|
|
6077
|
+
def _nonzero_static(x: Tensor, size: int):
|
|
6078
|
+
return x.nonzero_static(size=size)
|
|
6079
|
+
|
|
6080
|
+
|
|
6081
|
+
@torch.jit.script
|
|
6082
|
+
def _combine_similar_resps(resps: Tensor, keep_mask: Tensor, n_keep: int) -> Tensor:
|
|
6083
|
+
n_discard = resps.shape[1] - n_keep
|
|
6084
|
+
assert n_discard + n_keep == resps.shape[1]
|
|
6085
|
+
discard_mask = torch.logical_not(keep_mask)
|
|
6086
|
+
discard_ix = _nonzero_static(discard_mask, size=n_discard)[:, 0]
|
|
6087
|
+
keep_ix = _nonzero_static(keep_mask, size=n_keep)[:, 0]
|
|
6088
|
+
assert keep_ix.numel() + discard_ix.numel() == resps.shape[1]
|
|
6089
|
+
kept_resp = resps[:, keep_ix]
|
|
6090
|
+
discard_resp = resps[:, discard_ix]
|
|
6091
|
+
sim = discard_resp.T @ kept_resp
|
|
6092
|
+
match = sim.argmax(1)
|
|
6093
|
+
kept_resp[:, match] += discard_resp
|
|
6094
|
+
return kept_resp
|
|
6092
6095
|
|
|
6093
6096
|
|
|
6094
6097
|
def concatenate_scores(scoress: list[Scores]) -> Scores:
|
|
@@ -6201,10 +6204,7 @@ def mean_responsibilities(
|
|
|
6201
6204
|
rsum_batch.zero_()
|
|
6202
6205
|
|
|
6203
6206
|
nc = int(ncand[bix].item())
|
|
6204
|
-
|
|
6205
|
-
cii, cjj = (cand[i0:i1] >= 0).nonzero().T
|
|
6206
|
-
else:
|
|
6207
|
-
cii, cjj = (cand[i0:i1] >= 0).nonzero_static(size=nc).T
|
|
6207
|
+
cii, cjj = _nonzero_static(cand[i0:i1] >= 0, size=nc).T
|
|
6208
6208
|
|
|
6209
6209
|
c = cand[i0:i1][cii, cjj]
|
|
6210
6210
|
r = resp[i0:i1][cii, cjj].double()
|
|
@@ -6491,8 +6491,8 @@ def _sparsify_candidates(
|
|
|
6491
6491
|
assert cpos.any(dim=1).all()
|
|
6492
6492
|
if pnoid and static_size is not None:
|
|
6493
6493
|
assert cpos.sum() == static_size
|
|
6494
|
-
if static_size is not None
|
|
6495
|
-
spike_ixs, candidate_ixs = cpos
|
|
6494
|
+
if static_size is not None:
|
|
6495
|
+
spike_ixs, candidate_ixs = _nonzero_static(cpos, size=static_size).T
|
|
6496
6496
|
else:
|
|
6497
6497
|
spike_ixs, candidate_ixs = cpos.nonzero(as_tuple=True)
|
|
6498
6498
|
neighb_ixs = neighborhood_ids[spike_ixs]
|
|
@@ -23,7 +23,7 @@ from ..util.data_util import DARTsortSorting
|
|
|
23
23
|
from ..util.internal_config import ComputationConfig, unshifted_raw_template_cfg
|
|
24
24
|
from ..util.logging_util import progbar
|
|
25
25
|
from ..util.motion import MotionInfo
|
|
26
|
-
from ..util.py_util import
|
|
26
|
+
from ..util.py_util import ensure_path
|
|
27
27
|
from . import analysis, comparison, simkit
|
|
28
28
|
|
|
29
29
|
logger = getLogger(__name__)
|
|
@@ -406,7 +406,7 @@ def load_dartsort_step_sortings(
|
|
|
406
406
|
use, although its not a guarantee... h5 locking... need to figure it out.
|
|
407
407
|
"""
|
|
408
408
|
mtime_dt = mtime_gap_minutes * 60 if mtime_gap_minutes else 0
|
|
409
|
-
sorting_dir =
|
|
409
|
+
sorting_dir = ensure_path(sorting_dir, strict=True)
|
|
410
410
|
if detection_h5_path is None:
|
|
411
411
|
for dh5n in detection_h5_names:
|
|
412
412
|
detection_h5_path = cast(Path, sorting_dir / dh5n)
|
|
@@ -416,12 +416,12 @@ def load_dartsort_step_sortings(
|
|
|
416
416
|
age = time.time() - detection_h5_path.stat().st_mtime
|
|
417
417
|
if age < mtime_dt:
|
|
418
418
|
continue
|
|
419
|
-
h5s = [
|
|
419
|
+
h5s = [ensure_path(detection_h5_path)]
|
|
420
420
|
break
|
|
421
421
|
else:
|
|
422
422
|
h5s = []
|
|
423
423
|
else:
|
|
424
|
-
h5s = [
|
|
424
|
+
h5s = [ensure_path(detection_h5_path)]
|
|
425
425
|
|
|
426
426
|
for j in range(1, 100):
|
|
427
427
|
mh5 = sorting_dir / f"matching{j}.h5"
|
|
@@ -430,7 +430,7 @@ def load_dartsort_step_sortings(
|
|
|
430
430
|
if mtime_dt:
|
|
431
431
|
if time.time() - mh5.stat().st_mtime < mtime_dt:
|
|
432
432
|
break
|
|
433
|
-
h5s.append(
|
|
433
|
+
h5s.append(ensure_path(mh5))
|
|
434
434
|
|
|
435
435
|
# let's check that there is at least something to do...
|
|
436
436
|
labels_npys = sorting_dir.glob("*_labels.npy")
|
|
@@ -464,9 +464,9 @@ def load_dartsort_step_sortings(
|
|
|
464
464
|
)
|
|
465
465
|
if no_npys and st0.labels is not None:
|
|
466
466
|
if hasattr(st0, "template_inds"):
|
|
467
|
-
yield(
|
|
467
|
+
yield (
|
|
468
468
|
name_formatter(f"{h5.stem}_template"),
|
|
469
|
-
st0.ephemeral_replace(labels=st0.template_inds)
|
|
469
|
+
st0.ephemeral_replace(labels=st0.template_inds),
|
|
470
470
|
)
|
|
471
471
|
yield name_formatter(h5.stem), st0
|
|
472
472
|
continue
|
|
@@ -20,8 +20,8 @@ from ..templates import TemplateData
|
|
|
20
20
|
from ..util.data_util import (
|
|
21
21
|
DARTsortSorting,
|
|
22
22
|
divide_randomly,
|
|
23
|
+
ensure_path,
|
|
23
24
|
extract_random_snips,
|
|
24
|
-
resolve_path,
|
|
25
25
|
)
|
|
26
26
|
from ..util.job_util import ensure_computation_config
|
|
27
27
|
from ..util.logging_util import get_logger, progbar
|
|
@@ -108,7 +108,7 @@ def generate_simulation(
|
|
|
108
108
|
pass
|
|
109
109
|
|
|
110
110
|
if noise_recording_folder is not None:
|
|
111
|
-
noise_recording_folder =
|
|
111
|
+
noise_recording_folder = ensure_path(noise_recording_folder)
|
|
112
112
|
else:
|
|
113
113
|
assert noise_in_memory
|
|
114
114
|
duration_samples = int(duration_seconds * sampling_frequency)
|
|
@@ -146,7 +146,7 @@ def generate_simulation(
|
|
|
146
146
|
return
|
|
147
147
|
|
|
148
148
|
if folder is not None:
|
|
149
|
-
folder =
|
|
149
|
+
folder = ensure_path(folder)
|
|
150
150
|
else:
|
|
151
151
|
assert no_save
|
|
152
152
|
|
|
@@ -214,7 +214,7 @@ def generate_simulation(
|
|
|
214
214
|
|
|
215
215
|
|
|
216
216
|
def load_simulation(folder):
|
|
217
|
-
folder =
|
|
217
|
+
folder = ensure_path(folder, strict=True)
|
|
218
218
|
recording_dir = folder / "recording"
|
|
219
219
|
templates_npz = folder / "templates.npz"
|
|
220
220
|
sorting_h5 = folder / "dartsort_sorting.h5"
|
|
@@ -252,7 +252,9 @@ class InjectSpikesPreprocessor(BasePreprocessor):
|
|
|
252
252
|
**simulation_kwargs,
|
|
253
253
|
)
|
|
254
254
|
)
|
|
255
|
-
self.segment = cast(
|
|
255
|
+
self.segment = cast(
|
|
256
|
+
InjectSpikesPreprocessorSegment, self._recording_segments[0]
|
|
257
|
+
)
|
|
256
258
|
|
|
257
259
|
def basic_sorting(self) -> DARTsortSorting:
|
|
258
260
|
return self.segment.basic_sorting()
|
|
@@ -406,7 +408,9 @@ class InjectSpikesPreprocessor(BasePreprocessor):
|
|
|
406
408
|
|
|
407
409
|
# residual snippets
|
|
408
410
|
if n_residual_snips:
|
|
409
|
-
nrs_dset = h5.create_dataset(
|
|
411
|
+
nrs_dset = h5.create_dataset(
|
|
412
|
+
"n_residuals", data=np.zeros((), dtype=np.int64)
|
|
413
|
+
)
|
|
410
414
|
residual = h5.create_dataset(
|
|
411
415
|
"residual",
|
|
412
416
|
shape=(n_residual_snips, *self.segment.wf_shape),
|
|
@@ -486,7 +490,7 @@ class InjectSpikesPreprocessor(BasePreprocessor):
|
|
|
486
490
|
save_collidedness=False,
|
|
487
491
|
chunk_len_s=0.5,
|
|
488
492
|
):
|
|
489
|
-
folder =
|
|
493
|
+
folder = ensure_path(folder)
|
|
490
494
|
folder.mkdir(exist_ok=True)
|
|
491
495
|
recording_dir = folder / "recording"
|
|
492
496
|
templates_npz = folder / "templates.npz"
|
|
@@ -62,13 +62,13 @@ from .util.motion import MotionInfo, get_motion_info
|
|
|
62
62
|
from .util.noise_util import SpatialWhitener
|
|
63
63
|
from .util.peel_util import run_peeler
|
|
64
64
|
from .util.preprocess_util import preprocess
|
|
65
|
-
from .util.py_util import dartcopytree,
|
|
65
|
+
from .util.py_util import dartcopytree, ensure_path, timer
|
|
66
66
|
from .util.torch_util import cleanup_and_log_gpu_usage
|
|
67
67
|
|
|
68
68
|
logger = get_logger(__name__)
|
|
69
69
|
|
|
70
70
|
|
|
71
|
-
class
|
|
71
|
+
class DARTsortResult(TypedDict):
|
|
72
72
|
sorting: DARTsortSorting
|
|
73
73
|
"""Output spike trains."""
|
|
74
74
|
motion: MotionInfo
|
|
@@ -86,21 +86,25 @@ def dartsort(
|
|
|
86
86
|
dredge_motion_est: MotionEstimate | None = None,
|
|
87
87
|
overwrite=False,
|
|
88
88
|
):
|
|
89
|
-
"""This function runs a spike sorter called dartsort
|
|
89
|
+
"""This function runs a spike sorter called *dartsort*.
|
|
90
90
|
|
|
91
91
|
Parameters
|
|
92
92
|
---------
|
|
93
93
|
recording : BaseRecording
|
|
94
|
-
A SpikeInterface `BaseRecording`
|
|
94
|
+
A SpikeInterface `BaseRecording`
|
|
95
95
|
output_dir : str or Path
|
|
96
|
-
Folder where outputs are stored
|
|
96
|
+
Folder where outputs are stored. See the `work_in_tmpdir` and `tmpdir_parent`
|
|
97
|
+
configuration options to store intermediate data in a scratch folder and
|
|
98
|
+
then only save the final outputs here.
|
|
97
99
|
cfg : DARTsortUserConfig or DARTsortInternalConfig or str or Path
|
|
98
100
|
Your settings. Either create a `DARTsortUserConfig` directly in code, or
|
|
99
101
|
you can pass a string or Path pointing to a .toml file here.
|
|
100
102
|
si_motion : spikeinterface.core.Motion, optional
|
|
101
|
-
Allows users to pass their own external motion estimate.
|
|
103
|
+
Allows users to pass their own external motion estimate. If this is given,
|
|
104
|
+
the do_motion_estimation configuration flag is ignored and this object is
|
|
105
|
+
used.
|
|
102
106
|
dredge_motion_est : dredge.MotionEstimate, optional
|
|
103
|
-
|
|
107
|
+
As in `si_motion`.
|
|
104
108
|
overwrite : bool
|
|
105
109
|
Ignore and overwrite stored results, if any. Otherwise, dartsort will
|
|
106
110
|
try to resume from the last step that ran, or if it had finished then
|
|
@@ -108,13 +112,13 @@ def dartsort(
|
|
|
108
112
|
|
|
109
113
|
Returns
|
|
110
114
|
-------
|
|
111
|
-
results :
|
|
115
|
+
results : DARTsortResult
|
|
112
116
|
Dictionary of sorting results, with keys:
|
|
113
117
|
|
|
114
118
|
- "sorting": `DARTsortSorting`
|
|
115
119
|
- "motion": MotionInfo
|
|
116
120
|
"""
|
|
117
|
-
output_dir =
|
|
121
|
+
output_dir = ensure_path(output_dir)
|
|
118
122
|
output_dir.mkdir(exist_ok=True)
|
|
119
123
|
|
|
120
124
|
# convert cfg to internal format and store it for posterity
|
|
@@ -131,7 +135,7 @@ def dartsort(
|
|
|
131
135
|
if cfg.work_in_tmpdir:
|
|
132
136
|
with TemporaryDirectory(prefix="dartsort", dir=cfg.tmpdir_parent) as work_dir:
|
|
133
137
|
# copy files and possibly recording to temporary directory
|
|
134
|
-
work_dir =
|
|
138
|
+
work_dir = ensure_path(work_dir)
|
|
135
139
|
logger.dartsortdebug(f"Working in {work_dir}, outputs to {output_dir}.")
|
|
136
140
|
recording, work_dir = ds_all_to_workdir(
|
|
137
141
|
internal_cfg=cfg,
|
|
@@ -281,6 +285,7 @@ def _dartsort_impl(
|
|
|
281
285
|
sampling_cfg=cfg.peeler_sampling_cfg,
|
|
282
286
|
waveform_cfg=cfg.waveform_cfg,
|
|
283
287
|
overwrite=overwrite,
|
|
288
|
+
_saving_intermediates=cfg.save_intermediate_features,
|
|
284
289
|
)
|
|
285
290
|
ret["motion"] = motion
|
|
286
291
|
ds_save_motion(motion, output_dir, work_dir, overwrite)
|
|
@@ -301,6 +306,7 @@ def _dartsort_impl(
|
|
|
301
306
|
sorting,
|
|
302
307
|
motion=motion,
|
|
303
308
|
refinement_cfgs=r_cfgs,
|
|
309
|
+
computation_cfg=cfg.computation_cfg,
|
|
304
310
|
clustering_cfg=cfg.clustering_cfg,
|
|
305
311
|
clustering_features_cfg=cfg.clustering_features_cfg,
|
|
306
312
|
_save_cfg=cfg,
|
|
@@ -372,6 +378,7 @@ def _dartsort_impl(
|
|
|
372
378
|
recording=recording,
|
|
373
379
|
sorting=sorting,
|
|
374
380
|
motion=motion,
|
|
381
|
+
computation_cfg=cfg.computation_cfg,
|
|
375
382
|
refinement_cfgs=step_ref_cfgs,
|
|
376
383
|
clustering_cfg=step_clus_cfg,
|
|
377
384
|
clustering_features_cfg=step_clfeat_cfg,
|
|
@@ -390,7 +397,7 @@ def _dartsort_impl(
|
|
|
390
397
|
|
|
391
398
|
# finally handle scratch directory and delete intermediate files if requested
|
|
392
399
|
if work_dir is not None:
|
|
393
|
-
orig_h5_path =
|
|
400
|
+
orig_h5_path = ensure_path(sorting.parent_h5_path, strict=True)
|
|
394
401
|
final_h5_path = output_dir / orig_h5_path.name
|
|
395
402
|
assert final_h5_path.exists()
|
|
396
403
|
sorting.parent_h5_path = final_h5_path
|
|
@@ -483,7 +490,7 @@ def subtract(
|
|
|
483
490
|
hdf5_filename="subtraction.h5",
|
|
484
491
|
model_subdir="subtraction_models",
|
|
485
492
|
) -> DARTsortSorting | None:
|
|
486
|
-
output_dir =
|
|
493
|
+
output_dir = ensure_path(output_dir)
|
|
487
494
|
computation_cfg = ensure_computation_config(computation_cfg)
|
|
488
495
|
check_recording(recording)
|
|
489
496
|
subtraction_peeler = SubtractionPeeler.from_config(
|
|
@@ -544,7 +551,7 @@ def match(
|
|
|
544
551
|
template_denoising_tsvd=None,
|
|
545
552
|
whitener: SpatialWhitener | None = None,
|
|
546
553
|
) -> DARTsortSorting:
|
|
547
|
-
output_dir =
|
|
554
|
+
output_dir = ensure_path(output_dir)
|
|
548
555
|
model_dir = output_dir / model_subdir
|
|
549
556
|
computation_cfg = ensure_computation_config(computation_cfg)
|
|
550
557
|
|
|
@@ -632,7 +639,7 @@ def grab(
|
|
|
632
639
|
model_subdir="grab_models",
|
|
633
640
|
computation_cfg: ComputationConfig | None = None,
|
|
634
641
|
) -> DARTsortSorting:
|
|
635
|
-
output_dir =
|
|
642
|
+
output_dir = ensure_path(output_dir)
|
|
636
643
|
grabber = GrabAndFeaturize.from_config(
|
|
637
644
|
sorting=sorting,
|
|
638
645
|
recording=recording,
|
|
@@ -672,7 +679,7 @@ def threshold(
|
|
|
672
679
|
model_subdir="threshold_models",
|
|
673
680
|
computation_cfg: ComputationConfig | None = None,
|
|
674
681
|
) -> DARTsortSorting:
|
|
675
|
-
output_dir =
|
|
682
|
+
output_dir = ensure_path(output_dir)
|
|
676
683
|
computation_cfg = ensure_computation_config(computation_cfg)
|
|
677
684
|
thresholder = Threshold.from_config(
|
|
678
685
|
recording=recording,
|
|
@@ -10,7 +10,7 @@ from ...templates.template_util import CompressedUpsampledTemplates, LowRankTemp
|
|
|
10
10
|
from ...templates.templates import TemplateData
|
|
11
11
|
from ...util import job_util
|
|
12
12
|
from ...util.motion import MotionInfo
|
|
13
|
-
from ...util.py_util import
|
|
13
|
+
from ...util.py_util import ensure_path
|
|
14
14
|
from .matching_base import PconvBase
|
|
15
15
|
from .pairwise_util import compressed_convolve_to_h5
|
|
16
16
|
|
|
@@ -149,7 +149,7 @@ class CompressedPairwiseConv(PconvBase):
|
|
|
149
149
|
if computation_cfg is None:
|
|
150
150
|
computation_cfg = job_util.get_global_computation_config()
|
|
151
151
|
|
|
152
|
-
hdf5_filename =
|
|
152
|
+
hdf5_filename = ensure_path(hdf5_filename)
|
|
153
153
|
hdf5_filename.parent.mkdir(exist_ok=True)
|
|
154
154
|
|
|
155
155
|
# TODO: rewrite.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""A peeler implementing mean or median reduction for estimating template waveforms."""
|
|
2
|
+
|
|
2
3
|
from dataclasses import replace
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
from tempfile import TemporaryDirectory
|
|
@@ -31,7 +32,7 @@ from ..util.job_util import ensure_computation_config
|
|
|
31
32
|
from ..util.logging_util import get_logger
|
|
32
33
|
from ..util.motion import MotionInfo
|
|
33
34
|
from ..util.noise_util import SpatialWhitener
|
|
34
|
-
from ..util.py_util import
|
|
35
|
+
from ..util.py_util import ensure_path
|
|
35
36
|
from ..util.waveform_util import full_channel_index
|
|
36
37
|
from .grab import GrabAndFeaturize
|
|
37
38
|
|
|
@@ -58,6 +59,7 @@ class ReductionTemplateData(TemplateData):
|
|
|
58
59
|
computation_cfg: ComputationConfig | None = None,
|
|
59
60
|
show_progress: bool = True,
|
|
60
61
|
) -> TemplateData:
|
|
62
|
+
computation_cfg = ensure_computation_config(computation_cfg)
|
|
61
63
|
# subsample sorting
|
|
62
64
|
sorting = subsample_by_count_and_valid_time(
|
|
63
65
|
sorting,
|
|
@@ -75,24 +77,25 @@ class ReductionTemplateData(TemplateData):
|
|
|
75
77
|
motion=motion,
|
|
76
78
|
waveform_cfg=waveform_cfg,
|
|
77
79
|
template_cfg=template_cfg,
|
|
80
|
+
computation_cfg=computation_cfg,
|
|
78
81
|
whitener=whitener,
|
|
79
82
|
tsvd=tsvd,
|
|
80
83
|
)
|
|
81
84
|
|
|
82
85
|
# TODO: file not always needed
|
|
83
|
-
computation_cfg = ensure_computation_config(computation_cfg)
|
|
84
86
|
if template_cfg.reduction == "mean":
|
|
85
87
|
# TODO: reducer doesn't work in parallel when gathering means, go to single job
|
|
86
88
|
computation_cfg = ComputationConfig(
|
|
87
89
|
device=computation_cfg.actual_device().type,
|
|
88
90
|
n_jobs_small=computation_cfg.n_jobs_small,
|
|
91
|
+
tmpdir_parent=computation_cfg.tmpdir_parent,
|
|
89
92
|
)
|
|
90
93
|
with TemporaryDirectory(
|
|
91
94
|
prefix="dartsorttemplates",
|
|
92
95
|
ignore_cleanup_errors=True,
|
|
93
96
|
dir=computation_cfg.tmpdir_parent,
|
|
94
97
|
) as tdir:
|
|
95
|
-
tdir =
|
|
98
|
+
tdir = ensure_path(tdir)
|
|
96
99
|
h5p = tdir / "tmp.h5"
|
|
97
100
|
p.load_or_fit_and_save_models(tdir / "models")
|
|
98
101
|
if template_cfg.denoising_method == "none" and not template_cfg.use_svd:
|
|
@@ -190,6 +193,7 @@ class TemplateReduction(GrabAndFeaturize):
|
|
|
190
193
|
sorting: DARTsortSorting,
|
|
191
194
|
waveform_cfg: WaveformConfig,
|
|
192
195
|
template_cfg: TemplateConfig,
|
|
196
|
+
computation_cfg: ComputationConfig,
|
|
193
197
|
whitener: SpatialWhitener | None = None,
|
|
194
198
|
):
|
|
195
199
|
# geom processing
|
|
@@ -228,6 +232,7 @@ class TemplateReduction(GrabAndFeaturize):
|
|
|
228
232
|
motion=motion,
|
|
229
233
|
template_cfg=template_cfg,
|
|
230
234
|
waveform_cfg=waveform_cfg,
|
|
235
|
+
computation_cfg=computation_cfg,
|
|
231
236
|
)
|
|
232
237
|
assert tsvd.components_.shape[0] == template_cfg.denoising_rank
|
|
233
238
|
tsvd = FullProbeTemporalPCAEmbedder.from_sklearn(
|
|
@@ -26,7 +26,7 @@ from ..util.job_util import ensure_computation_config
|
|
|
26
26
|
from ..util.logging_util import get_logger
|
|
27
27
|
from ..util.motion import MotionInfo
|
|
28
28
|
from ..util.noise_util import SpatialWhitener
|
|
29
|
-
from ..util.py_util import
|
|
29
|
+
from ..util.py_util import ensure_path
|
|
30
30
|
from ..util.spiketorch import ptp
|
|
31
31
|
from . import TemplateData, realign
|
|
32
32
|
from .templib import fit_tsvd, pca_from_templates, quick_mean_templates
|
|
@@ -59,7 +59,7 @@ def estimate_template_library(
|
|
|
59
59
|
) -> tuple[DARTsortSorting, TemplateData]:
|
|
60
60
|
"""Postprocess spike train and estimate a TemplateData."""
|
|
61
61
|
if template_npz_path is not None:
|
|
62
|
-
template_npz_path =
|
|
62
|
+
template_npz_path = ensure_path(template_npz_path)
|
|
63
63
|
if template_npz_path.exists():
|
|
64
64
|
return sorting, TemplateData.from_npz(template_npz_path)
|
|
65
65
|
|
|
@@ -192,7 +192,7 @@ class AmortizedLocalization(BaseWaveformFeaturizer):
|
|
|
192
192
|
a0 = masks * pred_amps_alpha1
|
|
193
193
|
numer = a0.mul(obs_amps).sum(dim=1)
|
|
194
194
|
denom = a0.square().sum(dim=1)
|
|
195
|
-
alphas = numer.div_(denom)
|
|
195
|
+
alphas = numer.div_(denom + 1e-6)
|
|
196
196
|
if return_pred:
|
|
197
197
|
return alphas, alphas.unsqueeze(1) * pred_amps_alpha1
|
|
198
198
|
return alphas
|
|
@@ -203,13 +203,13 @@ class AmortizedLocalization(BaseWaveformFeaturizer):
|
|
|
203
203
|
if self.localization_model == "gaussian":
|
|
204
204
|
pred_amps_alpha1 = dists.square().mul(-2).exp()
|
|
205
205
|
else:
|
|
206
|
-
pred_amps_alpha1 = 1.0 / dists
|
|
206
|
+
pred_amps_alpha1 = 1.0 / (dists + 1e-6)
|
|
207
207
|
alphas, pred_amps = self.get_alphas(
|
|
208
208
|
obs_amps, pred_amps_alpha1, masks, return_pred=True
|
|
209
209
|
)
|
|
210
210
|
else:
|
|
211
211
|
alphas = F.softplus(z[:, 3])
|
|
212
|
-
pred_amps = alphas.unsqueeze(1) / dists
|
|
212
|
+
pred_amps = alphas.unsqueeze(1) / (dists + 1e-6)
|
|
213
213
|
|
|
214
214
|
return alphas, pred_amps
|
|
215
215
|
|
|
@@ -13,7 +13,7 @@ from ..util.internal_config import (
|
|
|
13
13
|
WaveformConfig,
|
|
14
14
|
)
|
|
15
15
|
from ..util.logging_util import get_logger
|
|
16
|
-
from ..util.py_util import
|
|
16
|
+
from ..util.py_util import ensure_path
|
|
17
17
|
from ..util.waveform_util import assert_all_finite_in_probe
|
|
18
18
|
from .transform_base import BaseWaveformFeaturizer, BaseWaveformModule
|
|
19
19
|
|
|
@@ -297,7 +297,7 @@ class WaveformPipeline(torch.nn.Module):
|
|
|
297
297
|
del waveforms
|
|
298
298
|
|
|
299
299
|
if hdf5_filename is not None:
|
|
300
|
-
hdf5_filename =
|
|
300
|
+
hdf5_filename = ensure_path(hdf5_filename, strict=True)
|
|
301
301
|
|
|
302
302
|
if self.safe:
|
|
303
303
|
assert torch.is_tensor(features["waveforms"])
|
|
@@ -377,7 +377,7 @@ class WaveformPipeline(torch.nn.Module):
|
|
|
377
377
|
if up_to_index == 0:
|
|
378
378
|
return
|
|
379
379
|
|
|
380
|
-
hdf5_filename =
|
|
380
|
+
hdf5_filename = ensure_path(hdf5_filename, strict=True)
|
|
381
381
|
dev = self.device
|
|
382
382
|
|
|
383
383
|
# use sorting as a way to load all 1d features, which transformers
|
|
@@ -139,7 +139,7 @@ class TemplateWaveformReducer(BaseWaveformFeaturizer):
|
|
|
139
139
|
dev = computation_cfg.actual_device()
|
|
140
140
|
|
|
141
141
|
n_jobs, Executor, context, *_ = pool_from_cfg(
|
|
142
|
-
computation_cfg, check_local=True, small=True
|
|
142
|
+
computation_cfg, check_local=True, small=True, cpu=dev.type == 'cpu'
|
|
143
143
|
)
|
|
144
144
|
with Executor(
|
|
145
145
|
max_workers=n_jobs,
|