DARTsort 0.5.31__tar.gz → 0.5.33__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.31 → dartsort-0.5.33}/.github/workflows/docs.yml +7 -5
- {dartsort-0.5.31 → dartsort-0.5.33}/PKG-INFO +1 -1
- {dartsort-0.5.31 → dartsort-0.5.33}/environment.yml +1 -1
- {dartsort-0.5.31 → dartsort-0.5.33}/notebook/fit_and_batch_matching.py +100 -32
- {dartsort-0.5.31 → dartsort-0.5.33}/notebook/ibl_dandi_example.py +16 -18
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/__init__.py +2 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/agglomerate.py +17 -5
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/cluster_util.py +15 -2
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/clustering.py +3 -1
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/clustering_features.py +22 -3
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/kmeans.py +56 -3
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/mixture.py +1215 -529
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/refine_util.py +3 -3
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/config.py +38 -15
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/hybridkit.py +2 -1
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/noise_recording_tools.py +2 -2
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/simkit.py +2 -2
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/simlib.py +2 -2
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/main.py +26 -10
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/drifty.py +13 -2
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/peel_base.py +191 -102
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/reduction_template.py +164 -4
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/subtract.py +2 -2
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/threshold.py +8 -1
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/get_templates.py +2 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/postprocess_util.py +2 -1
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/template_util.py +2 -2
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/templates.py +4 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/amortized_localization.py +13 -2
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/decollider.py +6 -1
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/mixture_classifier.py +17 -16
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/pipeline.py +85 -88
- dartsort-0.5.33/src/dartsort/transform/reduction.py +590 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/supervised_denoiser.py +15 -2
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/temporal_pca.py +13 -6
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/transform_base.py +23 -8
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/vq_matched_filter.py +2 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/whiten.py +2 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/data_util.py +82 -17
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/drift_util.py +60 -9
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/internal_config.py +148 -41
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/interpolation_util.py +55 -19
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/main_util.py +8 -3
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/motion.py +41 -31
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/multiprocessing_util.py +12 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/nn_util.py +9 -6
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/peel_util.py +7 -2
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/preprocess_util.py +1 -1
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/py_util.py +11 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/spiketorch.py +126 -20
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/gt.py +1 -1
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/mixture.py +1 -1
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_config.py +6 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_dartsort.py +24 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_drift_util.py +36 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_drifty_matching.py +55 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_matching.py +3 -3
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_mixture_classifier.py +24 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_modeling.py +128 -18
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_templates.py +204 -1
- dartsort-0.5.31/src/dartsort/transform/reduction.py +0 -269
- {dartsort-0.5.31 → dartsort-0.5.33}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/.gitignore +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/README.md +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/docs/index.md +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/docs/main_api.md +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/notebook/.keep +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/pyproject.toml +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/setup.cfg +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/__main__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/analysis.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/comparison.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/peel_lib.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/peel/subtract_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/matched_filter_net.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/noise_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/analysis_plots.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/vismain.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/SOURCES.txt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/conftest.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_agglomerate.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_alignment.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_clustering.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_clustering_features.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_data_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_density.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_detect.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_hybridkit.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_linalg.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_multiprocessing_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_simkit.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_subtract.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_threshold.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_transform.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.31 → dartsort-0.5.33}/zensical.toml +0 -0
|
@@ -29,16 +29,18 @@ jobs:
|
|
|
29
29
|
# this is a "fine-grained token"
|
|
30
30
|
token: ${{ secrets.DARTSORT_GITHUB_IO }}
|
|
31
31
|
|
|
32
|
-
- name: Setup
|
|
33
|
-
uses:
|
|
32
|
+
- name: Setup micromamba
|
|
33
|
+
uses: mamba-org/setup-micromamba@v3
|
|
34
34
|
with:
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
environment-file: dartsort/environment.yml
|
|
36
|
+
environment-name: dartsort-docs
|
|
37
37
|
|
|
38
38
|
- name: pip install stuff
|
|
39
|
-
|
|
39
|
+
shell: bash -el {0}
|
|
40
|
+
run: python -m pip install -e "dartsort/[doc]"
|
|
40
41
|
|
|
41
42
|
- name: Zensical build
|
|
43
|
+
shell: bash -el {0}
|
|
42
44
|
run: pushd dartsort/ && zensical build --clean && popd
|
|
43
45
|
|
|
44
46
|
- name: overwrite site
|
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
# extension: .py
|
|
6
6
|
# format_name: percent
|
|
7
7
|
# format_version: '1.3'
|
|
8
|
-
# jupytext_version: 1.19.
|
|
8
|
+
# jupytext_version: 1.19.6
|
|
9
9
|
# kernelspec:
|
|
10
|
-
# display_name: Python [conda env:
|
|
10
|
+
# display_name: Python [conda env:a]
|
|
11
11
|
# language: python
|
|
12
|
-
# name: conda-env-
|
|
12
|
+
# name: conda-env-a-py
|
|
13
13
|
# ---
|
|
14
14
|
|
|
15
15
|
# %% [markdown]
|
|
@@ -31,18 +31,16 @@
|
|
|
31
31
|
# recording.
|
|
32
32
|
|
|
33
33
|
# %%
|
|
34
|
-
import
|
|
35
|
-
|
|
36
|
-
# %%
|
|
37
|
-
# dandi-related
|
|
38
|
-
from dandi.dandiapi import DandiAPIClient
|
|
39
|
-
import fsspec
|
|
34
|
+
import dataclasses
|
|
40
35
|
|
|
41
|
-
|
|
36
|
+
import matplotlib.pyplot as plt
|
|
42
37
|
import numpy as np
|
|
43
38
|
import spikeinterface.full as si
|
|
39
|
+
from dandi.dandiapi import DandiAPIClient
|
|
40
|
+
|
|
44
41
|
import dartsort
|
|
45
42
|
import dartsort.vis as dartvis
|
|
43
|
+
|
|
46
44
|
dartvis.set_plt_style()
|
|
47
45
|
|
|
48
46
|
# %% [markdown]
|
|
@@ -50,12 +48,10 @@ dartvis.set_plt_style()
|
|
|
50
48
|
|
|
51
49
|
# %%
|
|
52
50
|
# update this for your machine!
|
|
53
|
-
experiment_path = dartsort.ensure_path(
|
|
54
|
-
"~/scratch/ibl-dandi-example/", mkdir=True
|
|
55
|
-
)
|
|
51
|
+
experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
|
|
56
52
|
|
|
57
53
|
# %%
|
|
58
|
-
ds_dir = experiment_path / "dartsort"
|
|
54
|
+
ds_dir = experiment_path / "dartsort-chunked"
|
|
59
55
|
|
|
60
56
|
# %%
|
|
61
57
|
# IBL BWM dandiset, example recording
|
|
@@ -93,7 +89,7 @@ else:
|
|
|
93
89
|
print(rec0)
|
|
94
90
|
|
|
95
91
|
# slice 10 minutes near the end
|
|
96
|
-
#TODO better IBL sampling rate handling
|
|
92
|
+
# TODO better IBL sampling rate handling
|
|
97
93
|
rec = rec0
|
|
98
94
|
end_time = rec.get_end_time()
|
|
99
95
|
start_frame = rec.time_to_sample_index(end_time - 30 * 60)
|
|
@@ -103,36 +99,109 @@ else:
|
|
|
103
99
|
rec._sampling_frequency = 30_000
|
|
104
100
|
|
|
105
101
|
rec = dartsort.preprocess(rec, strategy="ibllike")
|
|
106
|
-
rec = rec.
|
|
102
|
+
rec = rec.save(folder=preprocessing_path, format="binary")
|
|
107
103
|
|
|
108
104
|
# %%
|
|
109
105
|
plt.imshow(
|
|
110
106
|
rec.get_traces(0, 10_000, 11_000).T,
|
|
111
|
-
aspect=
|
|
107
|
+
aspect="auto",
|
|
112
108
|
vmin=-5,
|
|
113
109
|
vmax=5,
|
|
114
|
-
interpolation=
|
|
110
|
+
interpolation="nearest",
|
|
115
111
|
)
|
|
116
|
-
plt.colorbar(shrink=0.5, label=
|
|
117
|
-
plt.xlabel(
|
|
118
|
-
plt.ylabel(
|
|
119
|
-
|
|
112
|
+
plt.colorbar(shrink=0.5, label="standardized voltage")
|
|
113
|
+
plt.xlabel("time (samples")
|
|
114
|
+
plt.ylabel("channels")
|
|
120
115
|
# %% [markdown]
|
|
121
116
|
# # Configuration
|
|
122
117
|
|
|
123
118
|
# %%
|
|
124
119
|
# use defaults, except for the pipeline change
|
|
125
|
-
cfg = dartsort.DARTsortUserConfig(
|
|
120
|
+
cfg = dartsort.DARTsortUserConfig()
|
|
126
121
|
|
|
127
122
|
# %%
|
|
128
123
|
# process into an internal config object for later use
|
|
129
124
|
internal_cfg = dartsort.to_internal_config(cfg)
|
|
130
125
|
|
|
131
126
|
# %% [markdown]
|
|
132
|
-
# #
|
|
127
|
+
# # Initial spike detection and motion tracking model fits
|
|
133
128
|
|
|
134
129
|
# %%
|
|
135
|
-
dartsort.dartsort(rec, ds_dir, cfg=cfg)
|
|
130
|
+
dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_motion_models_only=True))
|
|
131
|
+
|
|
132
|
+
# %% [markdown]
|
|
133
|
+
# # Chunked spike detection for motion tracking
|
|
134
|
+
#
|
|
135
|
+
# While dartsort itself only needs to detect spikes on a subsample of the recording
|
|
136
|
+
# in order to do clustering, motion tracking does require spikes from the whole
|
|
137
|
+
# recording. dartsort uses a cheap but good spike detection and localization pass for
|
|
138
|
+
# this, which is very fast, but users at the thousand hour scale still might want to
|
|
139
|
+
# parallelize it.
|
|
140
|
+
#
|
|
141
|
+
# This runs that cheap spike detection for motion tracking in 1-minute batches for demo
|
|
142
|
+
# purposes.
|
|
143
|
+
#
|
|
144
|
+
# Here, the `chunk_starts_samples` argument slices the recording for us.
|
|
145
|
+
# This is better in this case than using `.frame_slice` or `.time_slice` on the recording,
|
|
146
|
+
# since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
|
|
147
|
+
#
|
|
148
|
+
# If the initial detection already covered the whole recording, motion tracking uses its
|
|
149
|
+
# spikes and this step is skipped.
|
|
150
|
+
|
|
151
|
+
# %%
|
|
152
|
+
subtraction_sorting = dartsort.load(ds_dir / "subtraction.h5")
|
|
153
|
+
need_motion_peaks = cfg.do_motion_estimation and dartsort.motion_needs_peaks(
|
|
154
|
+
internal_cfg, rec, subtraction_sorting
|
|
155
|
+
)
|
|
156
|
+
need_motion_peaks
|
|
157
|
+
|
|
158
|
+
# %%
|
|
159
|
+
chunk_length_samples = internal_cfg.motion_estimation_cfg.threshold_cfg.chunk_length_samples
|
|
160
|
+
batch_length_samples = 60 * chunk_length_samples
|
|
161
|
+
duration_samples = rec.get_num_samples()
|
|
162
|
+
|
|
163
|
+
if need_motion_peaks:
|
|
164
|
+
for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, batch_length_samples)):
|
|
165
|
+
batch_end_samples = min(duration_samples, batch_start_samples + batch_length_samples)
|
|
166
|
+
res = dartsort.detect_for_motion(
|
|
167
|
+
recording=rec,
|
|
168
|
+
output_dir=ds_dir,
|
|
169
|
+
previous_sorting=subtraction_sorting,
|
|
170
|
+
hdf5_filename=f"motionthreshold_batch{batch_ix:05}.h5",
|
|
171
|
+
chunk_starts_samples=np.arange(batch_start_samples, batch_end_samples, chunk_length_samples),
|
|
172
|
+
motion_cfg=internal_cfg.motion_estimation_cfg,
|
|
173
|
+
computation_cfg=internal_cfg.motion_computation_cfg,
|
|
174
|
+
sampling_cfg=internal_cfg.peeler_sampling_cfg,
|
|
175
|
+
waveform_cfg=internal_cfg.waveform_cfg,
|
|
176
|
+
)
|
|
177
|
+
del res
|
|
178
|
+
|
|
179
|
+
# %% [markdown]
|
|
180
|
+
# Combine the output of those detections into one sorting, and check that together the
|
|
181
|
+
# batches covered the recording.
|
|
182
|
+
|
|
183
|
+
# %%
|
|
184
|
+
if need_motion_peaks and not (ds_dir / "motionthreshold.h5").exists():
|
|
185
|
+
motion_chunk_h5s = sorted(ds_dir.glob("motionthreshold_batch*.h5"))
|
|
186
|
+
dartsort.virtual_concatenate_sorting_h5s(motion_chunk_h5s, ds_dir / "motionthreshold.h5")
|
|
187
|
+
motion_sorting = dartsort.load(ds_dir / "motionthreshold.h5")
|
|
188
|
+
assert np.array_equal(
|
|
189
|
+
motion_sorting.load_dataset("chunk_starts_samples"),
|
|
190
|
+
np.arange(0, duration_samples, chunk_length_samples),
|
|
191
|
+
)
|
|
192
|
+
print(motion_sorting)
|
|
193
|
+
|
|
194
|
+
# %%
|
|
195
|
+
# !ls -lah {ds_dir}
|
|
196
|
+
|
|
197
|
+
# %% [markdown]
|
|
198
|
+
# ^ Notice all of the chunked motion files! But motionthreshold.h5 itself is tiny, because the concatenation is virtual.
|
|
199
|
+
|
|
200
|
+
# %% [markdown]
|
|
201
|
+
# # Resume: estimate motion, cluster, learn templates and the matching step classifier
|
|
202
|
+
|
|
203
|
+
# %%
|
|
204
|
+
dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_matching_models_only=True))
|
|
136
205
|
|
|
137
206
|
# %%
|
|
138
207
|
# !ls {ds_dir}
|
|
@@ -146,11 +215,8 @@ dartsort.dartsort(rec, ds_dir, cfg=cfg)
|
|
|
146
215
|
# %% [markdown]
|
|
147
216
|
# # Chunked template matching
|
|
148
217
|
#
|
|
149
|
-
# This runs template matching in 1-minute batches
|
|
150
|
-
#
|
|
151
|
-
# Here, `chunk_starts_samples` slices the recording for us.
|
|
152
|
-
# This might be better than using `.frame_slice` or `.time_slice` on the recording,
|
|
153
|
-
# since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
|
|
218
|
+
# This runs template matching in 1-minute batches, with the same loop over chunks
|
|
219
|
+
# as the motion tracking's spike detection above.
|
|
154
220
|
#
|
|
155
221
|
|
|
156
222
|
# %%
|
|
@@ -186,7 +252,9 @@ for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, 60 * 3
|
|
|
186
252
|
# which can be turned off for users who don't need them.
|
|
187
253
|
|
|
188
254
|
# %%
|
|
189
|
-
sorting = dartsort.concatenate_sortings(
|
|
255
|
+
sorting = dartsort.concatenate_sortings(
|
|
256
|
+
[dartsort.load(h5) for h5 in sorted(ds_dir.glob("matching1_batch*.h5"))]
|
|
257
|
+
)
|
|
190
258
|
|
|
191
259
|
# %%
|
|
192
260
|
sorting = dartsort.cluster(
|
|
@@ -199,7 +267,7 @@ sorting = dartsort.cluster(
|
|
|
199
267
|
|
|
200
268
|
# %%
|
|
201
269
|
# if you want to keep this...
|
|
202
|
-
sorting.save(ds_dir /
|
|
270
|
+
sorting.save(ds_dir / "dartsort_sorting.npz")
|
|
203
271
|
|
|
204
272
|
# %% [markdown]
|
|
205
273
|
# # Visualize
|
|
@@ -5,25 +5,26 @@
|
|
|
5
5
|
# extension: .py
|
|
6
6
|
# format_name: percent
|
|
7
7
|
# format_version: '1.3'
|
|
8
|
-
# jupytext_version: 1.19.
|
|
8
|
+
# jupytext_version: 1.19.6
|
|
9
9
|
# kernelspec:
|
|
10
|
-
# display_name: Python [conda env:
|
|
10
|
+
# display_name: Python [conda env:a]
|
|
11
11
|
# language: python
|
|
12
|
-
# name: conda-env-
|
|
12
|
+
# name: conda-env-a-py
|
|
13
13
|
# ---
|
|
14
14
|
|
|
15
15
|
# %%
|
|
16
16
|
import matplotlib.pyplot as plt
|
|
17
17
|
|
|
18
|
+
# %%
|
|
19
|
+
import spikeinterface.full as si
|
|
20
|
+
|
|
18
21
|
# %%
|
|
19
22
|
# dandi-related
|
|
20
23
|
from dandi.dandiapi import DandiAPIClient
|
|
21
|
-
import fsspec
|
|
22
24
|
|
|
23
|
-
# %%
|
|
24
|
-
import spikeinterface.full as si
|
|
25
25
|
import dartsort
|
|
26
26
|
import dartsort.vis as dartvis
|
|
27
|
+
|
|
27
28
|
dartvis.set_plt_style()
|
|
28
29
|
|
|
29
30
|
# %% [markdown]
|
|
@@ -31,9 +32,7 @@ dartvis.set_plt_style()
|
|
|
31
32
|
|
|
32
33
|
# %%
|
|
33
34
|
# update this for your machine!
|
|
34
|
-
experiment_path = dartsort.ensure_path(
|
|
35
|
-
"~/scratch/ibl-dandi-example/", mkdir=True
|
|
36
|
-
)
|
|
35
|
+
experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
|
|
37
36
|
|
|
38
37
|
# %%
|
|
39
38
|
# IBL BWM dandiset, example recording
|
|
@@ -71,7 +70,7 @@ else:
|
|
|
71
70
|
print(rec0)
|
|
72
71
|
|
|
73
72
|
# slice 10 minutes near the end
|
|
74
|
-
#TODO better IBL sampling rate handling
|
|
73
|
+
# TODO better IBL sampling rate handling
|
|
75
74
|
rec = rec0
|
|
76
75
|
end_time = rec.get_end_time()
|
|
77
76
|
start_frame = rec.time_to_sample_index(end_time - 30 * 60)
|
|
@@ -81,20 +80,19 @@ else:
|
|
|
81
80
|
rec._sampling_frequency = 30_000
|
|
82
81
|
|
|
83
82
|
rec = dartsort.preprocess(rec, strategy="ibllike")
|
|
84
|
-
rec = rec.
|
|
83
|
+
rec = rec.save(folder=preprocessing_path, format="binary")
|
|
85
84
|
|
|
86
85
|
# %%
|
|
87
86
|
plt.imshow(
|
|
88
87
|
rec.get_traces(0, 10_000, 11_000).T,
|
|
89
|
-
aspect=
|
|
88
|
+
aspect="auto",
|
|
90
89
|
vmin=-5,
|
|
91
90
|
vmax=5,
|
|
92
|
-
interpolation=
|
|
91
|
+
interpolation="nearest",
|
|
93
92
|
)
|
|
94
|
-
plt.colorbar(shrink=0.5, label=
|
|
95
|
-
plt.xlabel(
|
|
96
|
-
plt.ylabel(
|
|
97
|
-
|
|
93
|
+
plt.colorbar(shrink=0.5, label="standardized voltage")
|
|
94
|
+
plt.xlabel("time (samples")
|
|
95
|
+
plt.ylabel("channels")
|
|
98
96
|
# %% [markdown]
|
|
99
97
|
# ## Run *dartsort*
|
|
100
98
|
|
|
@@ -106,6 +104,6 @@ ds_res = dartsort.dartsort(rec, ds_dir)
|
|
|
106
104
|
# # Visualize
|
|
107
105
|
|
|
108
106
|
# %%
|
|
109
|
-
dartvis.scatter_spike_features(sorting=ds_res["sorting"])
|
|
107
|
+
dartvis.scatter_spike_features(sorting=ds_res["sorting"])
|
|
110
108
|
|
|
111
109
|
# %%
|
|
@@ -63,6 +63,7 @@ from .util.data_util import (
|
|
|
63
63
|
sorting_from_spikeinterface,
|
|
64
64
|
try_get_denoising_pipeline,
|
|
65
65
|
try_get_model_dir,
|
|
66
|
+
virtual_concatenate_sorting_h5s,
|
|
66
67
|
)
|
|
67
68
|
from .util.drift_util import registered_geometry
|
|
68
69
|
from .util.internal_config import *
|
|
@@ -80,6 +81,7 @@ from .util.logging_util import (
|
|
|
80
81
|
get_logger,
|
|
81
82
|
set_log_level,
|
|
82
83
|
)
|
|
84
|
+
from .util.main_util import motion_needs_peaks
|
|
83
85
|
from .util.motion import (
|
|
84
86
|
MotionInfo,
|
|
85
87
|
detect_for_motion,
|
|
@@ -561,6 +561,7 @@ def template_distances(
|
|
|
561
561
|
sorting=sorting,
|
|
562
562
|
motion=motion,
|
|
563
563
|
radius=template_merge_cfg.weighted_dist_radius,
|
|
564
|
+
registered_geom=template_data.registered_geom,
|
|
564
565
|
)
|
|
565
566
|
spatial_weights = radial_weights(radial_counts)
|
|
566
567
|
assert np.isfinite(spatial_weights).all()
|
|
@@ -876,23 +877,34 @@ def bimod_stats(h):
|
|
|
876
877
|
return a, b
|
|
877
878
|
|
|
878
879
|
|
|
879
|
-
def radial_spike_counts(
|
|
880
|
+
def radial_spike_counts(
|
|
881
|
+
sorting: DARTsortSorting,
|
|
882
|
+
motion: MotionInfo,
|
|
883
|
+
radius: float,
|
|
884
|
+
registered_geom: np.ndarray | None = None,
|
|
885
|
+
):
|
|
880
886
|
assert sorting.labels is not None
|
|
887
|
+
if registered_geom is None:
|
|
888
|
+
registered_geom = motion.rgeom
|
|
881
889
|
kept = np.flatnonzero(sorting.labels >= 0)
|
|
882
890
|
|
|
883
891
|
# which reg chans do the spikes land on?
|
|
884
892
|
x, z = motion.geom[sorting.channels[kept]].T
|
|
885
893
|
cc = closest_registered_channels(
|
|
886
|
-
times_seconds=sorting.times_seconds[kept],
|
|
894
|
+
times_seconds=sorting.times_seconds[kept],
|
|
895
|
+
x=x,
|
|
896
|
+
z_abs=z,
|
|
897
|
+
motion=motion,
|
|
898
|
+
registered_geom=registered_geom,
|
|
887
899
|
)
|
|
888
900
|
|
|
889
901
|
# count by label
|
|
890
902
|
ll = sorting.labels[kept]
|
|
891
|
-
counts = np.zeros((ll.max() + 1,
|
|
903
|
+
counts = np.zeros((ll.max() + 1, registered_geom.shape[0]), dtype=np.int64)
|
|
892
904
|
np.add.at(counts, (ll, cc), 1)
|
|
893
905
|
|
|
894
906
|
# get radial neighborhoods
|
|
895
|
-
ci = make_channel_index(
|
|
907
|
+
ci = make_channel_index(registered_geom, radius, to_torch=False)
|
|
896
908
|
|
|
897
909
|
# puff out with radial neighborhood and sum up the counts by label
|
|
898
910
|
radial_counts = np.zeros(counts.shape)
|
|
@@ -901,7 +913,7 @@ def radial_spike_counts(sorting: DARTsortSorting, motion: MotionInfo, radius: fl
|
|
|
901
913
|
ii = np.flatnonzero(row)
|
|
902
914
|
for channel, value in zip(ii, row[ii], strict=True):
|
|
903
915
|
cixs = ci[channel]
|
|
904
|
-
cixs = cixs[cixs <
|
|
916
|
+
cixs = cixs[cixs < registered_geom.shape[0]]
|
|
905
917
|
radial_counts[uu, cixs] += value
|
|
906
918
|
|
|
907
919
|
return radial_counts
|
|
@@ -428,7 +428,13 @@ def reorder_by_depth(
|
|
|
428
428
|
|
|
429
429
|
|
|
430
430
|
def closest_registered_channels(
|
|
431
|
-
*,
|
|
431
|
+
*,
|
|
432
|
+
times_seconds,
|
|
433
|
+
x,
|
|
434
|
+
z_abs,
|
|
435
|
+
z_reg=None,
|
|
436
|
+
motion: MotionInfo,
|
|
437
|
+
registered_geom: np.ndarray | None = None,
|
|
432
438
|
) -> np.ndarray:
|
|
433
439
|
"""Assign spikes to the drift-extended channel closest to their registered position"""
|
|
434
440
|
if z_reg is None:
|
|
@@ -436,7 +442,14 @@ def closest_registered_channels(
|
|
|
436
442
|
z_reg = motion.correct_s(times_seconds, z_abs)
|
|
437
443
|
reg_pos = np.c_[x, z_reg]
|
|
438
444
|
|
|
439
|
-
|
|
445
|
+
if registered_geom is None or (
|
|
446
|
+
registered_geom.shape == motion.rgeom.shape
|
|
447
|
+
and np.allclose(registered_geom, motion.rgeom)
|
|
448
|
+
):
|
|
449
|
+
kdt = motion.rgeom_kdt
|
|
450
|
+
else:
|
|
451
|
+
kdt = KDTree(registered_geom)
|
|
452
|
+
_, reg_channels = kdt.query(reg_pos)
|
|
440
453
|
reg_channels = np.atleast_1d(reg_channels)
|
|
441
454
|
|
|
442
455
|
return reg_channels
|
|
@@ -59,7 +59,9 @@ def get_clusterer(
|
|
|
59
59
|
else:
|
|
60
60
|
clus_strategy = "none"
|
|
61
61
|
|
|
62
|
-
saving_labels =
|
|
62
|
+
saving_labels = save_labels_dir is not None and (
|
|
63
|
+
save_cfg is None or save_cfg.save_intermediate_labels
|
|
64
|
+
)
|
|
63
65
|
if not saving_labels:
|
|
64
66
|
save_labels_dir = save_cfg = None
|
|
65
67
|
|
|
@@ -243,7 +243,7 @@ class StableWaveformFeatures:
|
|
|
243
243
|
# n, nc_extract
|
|
244
244
|
channels: Tensor
|
|
245
245
|
# n, rank, nc_extract
|
|
246
|
-
features: Tensor
|
|
246
|
+
features: Tensor | np.ndarray
|
|
247
247
|
neighborhoods: SpikeNeighborhoods
|
|
248
248
|
erp: StableFeaturesInterpolator
|
|
249
249
|
|
|
@@ -292,6 +292,9 @@ class StableWaveformFeatures:
|
|
|
292
292
|
trim_to_rank=clustering_features_cfg.feature_rank,
|
|
293
293
|
params=clustering_features_cfg.interp_params.normalize(),
|
|
294
294
|
device=computation_cfg.actual_device(),
|
|
295
|
+
store_on_disk=computation_cfg.low_memory,
|
|
296
|
+
tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
|
|
297
|
+
nan_to_num=True,
|
|
295
298
|
)
|
|
296
299
|
else:
|
|
297
300
|
assert sorting.parent_h5_path is not None
|
|
@@ -308,13 +311,18 @@ class StableWaveformFeatures:
|
|
|
308
311
|
trim_to_rank=clustering_features_cfg.feature_rank,
|
|
309
312
|
params=clustering_features_cfg.interp_params.normalize(),
|
|
310
313
|
device=computation_cfg.actual_device(),
|
|
314
|
+
store_on_disk=computation_cfg.low_memory,
|
|
315
|
+
tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
|
|
316
|
+
nan_to_num=True,
|
|
311
317
|
)
|
|
312
318
|
|
|
313
|
-
logger.dartsortdebug(
|
|
319
|
+
logger.dartsortdebug(
|
|
320
|
+
f"StableWaveformFeatures {features.shape=} {computation_cfg.low_memory=}."
|
|
321
|
+
)
|
|
314
322
|
|
|
315
323
|
return cls(
|
|
316
324
|
channels=channels,
|
|
317
|
-
features=
|
|
325
|
+
features=features,
|
|
318
326
|
neighborhoods=spike_neighborhoods,
|
|
319
327
|
erp=erp,
|
|
320
328
|
)
|
|
@@ -323,6 +331,17 @@ class StableWaveformFeatures:
|
|
|
323
331
|
# -- helpers
|
|
324
332
|
|
|
325
333
|
|
|
334
|
+
def _read_ix_maybe_mmap(x: Tensor | np.ndarray, ix: Tensor | np.ndarray | slice) -> Tensor:
|
|
335
|
+
if isinstance(x, Tensor):
|
|
336
|
+
return x[ix]
|
|
337
|
+
if isinstance(ix, slice):
|
|
338
|
+
return torch.from_numpy(np.array(x[ix]))
|
|
339
|
+
if isinstance(ix, Tensor):
|
|
340
|
+
ix = ix.numpy(force=True)
|
|
341
|
+
assert ix.ndim == 1
|
|
342
|
+
return torch.from_numpy(x[ix])
|
|
343
|
+
|
|
344
|
+
|
|
326
345
|
def _interpolate_main_channel_features(
|
|
327
346
|
*,
|
|
328
347
|
sorting: DARTsortSorting,
|
|
@@ -15,6 +15,7 @@ except ImportError:
|
|
|
15
15
|
HAVE_CUPY = False
|
|
16
16
|
|
|
17
17
|
|
|
18
|
+
from ..util.internal_config import SplitKmeansppMetric
|
|
18
19
|
from ..util.logging_util import DARTSORTDEBUG, get_logger, progrange
|
|
19
20
|
from ..util.sparse_util import (
|
|
20
21
|
coo_to_cupy,
|
|
@@ -23,7 +24,11 @@ from ..util.sparse_util import (
|
|
|
23
24
|
logsumexp_coo,
|
|
24
25
|
sparse_centroid_distsq,
|
|
25
26
|
)
|
|
26
|
-
from ..util.spiketorch import
|
|
27
|
+
from ..util.spiketorch import (
|
|
28
|
+
scaled_normeuc_cdist_from_dots_,
|
|
29
|
+
spawn_torch_rg,
|
|
30
|
+
sqeuc_cdist_known_norm,
|
|
31
|
+
)
|
|
27
32
|
|
|
28
33
|
logger = get_logger(__name__)
|
|
29
34
|
|
|
@@ -631,6 +636,8 @@ def batched_kmeans(
|
|
|
631
636
|
with_labels=True,
|
|
632
637
|
with_proportions=True,
|
|
633
638
|
beta: float = 1.0,
|
|
639
|
+
kmeanspp_metric: SplitKmeansppMetric = "euc",
|
|
640
|
+
scale_dist_args: tuple[float, float, float] | None = None,
|
|
634
641
|
) -> KMeansResult:
|
|
635
642
|
"""
|
|
636
643
|
Compared to above:
|
|
@@ -639,6 +646,7 @@ def batched_kmeans(
|
|
|
639
646
|
- no weights allowed for now
|
|
640
647
|
- kmeanspp always random initial
|
|
641
648
|
- n_iter > 0
|
|
649
|
+
- kmeanspp_metric for seeding only
|
|
642
650
|
"""
|
|
643
651
|
k = n_components
|
|
644
652
|
del n_components
|
|
@@ -660,9 +668,17 @@ def batched_kmeans(
|
|
|
660
668
|
centroid_ixs = torch.full((k, n_kmeanspps), n, dtype=torch.long, device=dev)
|
|
661
669
|
centroid_ixs[0] = torch.randint(n, size=(n_kmeanspps,), device=dev, generator=gen)
|
|
662
670
|
dists = X.new_empty((n_kmeanspps, n))
|
|
671
|
+
if kmeanspp_metric == "euc":
|
|
672
|
+
seed_bufs = None
|
|
673
|
+
elif kmeanspp_metric == "scaled_normeuc":
|
|
674
|
+
seed_bufs = X.new_empty((4, n_kmeanspps, n))
|
|
675
|
+
else:
|
|
676
|
+
panic(kmeanspp_metric)
|
|
663
677
|
Y = X[centroid_ixs[0]]
|
|
664
678
|
Ynormsq = Xnormsq[centroid_ixs[0]]
|
|
665
|
-
dists =
|
|
679
|
+
dists = _kmeanspp_distsq(
|
|
680
|
+
Y, Ynormsq, X, Xnormsq, dists, seed_bufs, kmeanspp_metric, scale_dist_args
|
|
681
|
+
)
|
|
666
682
|
|
|
667
683
|
# -- kmeanspp stage: loop
|
|
668
684
|
# buf for random sampling with Gumbel trick and new distance storage
|
|
@@ -680,8 +696,11 @@ def batched_kmeans(
|
|
|
680
696
|
torch.index_select(Xnormsq, dim=0, index=cix_j, out=Ynormsq)
|
|
681
697
|
|
|
682
698
|
# update distances
|
|
683
|
-
newdists =
|
|
699
|
+
newdists = _kmeanspp_distsq(
|
|
700
|
+
Y, Ynormsq, X, Xnormsq, _buf, seed_bufs, kmeanspp_metric, scale_dist_args
|
|
701
|
+
)
|
|
684
702
|
torch.minimum(dists, newdists, out=dists)
|
|
703
|
+
del seed_bufs
|
|
685
704
|
|
|
686
705
|
# -- kmeanspp finish: pick best by phi
|
|
687
706
|
if kmeanspp_seeds_per_try > 1:
|
|
@@ -754,6 +773,40 @@ def batched_kmeans(
|
|
|
754
773
|
return KMeansResult(centroids=Y, responsibilities=e, labels=labels, dists=dists)
|
|
755
774
|
|
|
756
775
|
|
|
776
|
+
def _kmeanspp_distsq(
|
|
777
|
+
Y: Tensor,
|
|
778
|
+
Ynormsq: Tensor,
|
|
779
|
+
X: Tensor,
|
|
780
|
+
Xnormsq: Tensor,
|
|
781
|
+
out: Tensor,
|
|
782
|
+
bufs: Tensor | None,
|
|
783
|
+
metric: SplitKmeansppMetric,
|
|
784
|
+
scale_dist_args: tuple[float, float, float] | None,
|
|
785
|
+
) -> Tensor:
|
|
786
|
+
if metric == "euc":
|
|
787
|
+
assert bufs is None
|
|
788
|
+
return sqeuc_cdist_known_norm(Y, Ynormsq, X, Xnormsq, out)
|
|
789
|
+
elif metric == "scaled_normeuc":
|
|
790
|
+
assert bufs is not None
|
|
791
|
+
assert scale_dist_args is not None
|
|
792
|
+
scale_std, scale_min, scale_max = scale_dist_args
|
|
793
|
+
dots = torch.mm(Y, X.T, out=bufs[0])
|
|
794
|
+
out = scaled_normeuc_cdist_from_dots_(
|
|
795
|
+
dots,
|
|
796
|
+
Ynormsq,
|
|
797
|
+
Xnormsq,
|
|
798
|
+
out=out,
|
|
799
|
+
bufs=bufs[1:],
|
|
800
|
+
scale_var=scale_std**2,
|
|
801
|
+
scale_min=scale_min,
|
|
802
|
+
scale_max=scale_max,
|
|
803
|
+
)
|
|
804
|
+
assert out.isfinite().all()
|
|
805
|
+
return out.square_()
|
|
806
|
+
else:
|
|
807
|
+
panic(metric)
|
|
808
|
+
|
|
809
|
+
|
|
757
810
|
def truncated_kmeans_from_labels(
|
|
758
811
|
X: Tensor | np.ndarray,
|
|
759
812
|
labels: Tensor | np.ndarray,
|