DARTsort 0.5.32__tar.gz → 0.5.33__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.32 → dartsort-0.5.33}/.github/workflows/docs.yml +7 -5
- {dartsort-0.5.32 → dartsort-0.5.33}/PKG-INFO +1 -1
- {dartsort-0.5.32 → dartsort-0.5.33}/environment.yml +1 -1
- {dartsort-0.5.32 → dartsort-0.5.33}/notebook/fit_and_batch_matching.py +100 -32
- {dartsort-0.5.32 → dartsort-0.5.33}/notebook/ibl_dandi_example.py +16 -18
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/__init__.py +2 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/clustering_features.py +22 -3
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/mixture.py +52 -34
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/refine_util.py +3 -3
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/config.py +3 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/main.py +12 -6
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/reduction.py +23 -22
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/data_util.py +80 -16
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/internal_config.py +5 -1
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/interpolation_util.py +22 -5
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/motion.py +41 -31
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/py_util.py +11 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_modeling.py +25 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/.gitignore +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/README.md +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/docs/index.md +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/docs/main_api.md +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/notebook/.keep +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/pyproject.toml +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/setup.cfg +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/__main__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/agglomerate.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/cluster_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/clustering.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/kmeans.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/analysis.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/comparison.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/hybridkit.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/simkit.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/peel_base.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/peel_lib.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/subtract.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/subtract_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/decollider.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/matched_filter_net.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/mixture_classifier.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/pipeline.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/transform_base.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/vq_matched_filter.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/main_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/noise_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/spiketorch.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/analysis_plots.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/mixture.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/vismain.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/SOURCES.txt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/conftest.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_agglomerate.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_alignment.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_clustering.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_clustering_features.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_config.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_data_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_density.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_detect.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_hybridkit.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_linalg.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_matching.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_multiprocessing_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_simkit.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_subtract.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_templates.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_threshold.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_transform.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.33}/zensical.toml +0 -0
|
@@ -29,16 +29,18 @@ jobs:
|
|
|
29
29
|
# this is a "fine-grained token"
|
|
30
30
|
token: ${{ secrets.DARTSORT_GITHUB_IO }}
|
|
31
31
|
|
|
32
|
-
- name: Setup
|
|
33
|
-
uses:
|
|
32
|
+
- name: Setup micromamba
|
|
33
|
+
uses: mamba-org/setup-micromamba@v3
|
|
34
34
|
with:
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
environment-file: dartsort/environment.yml
|
|
36
|
+
environment-name: dartsort-docs
|
|
37
37
|
|
|
38
38
|
- name: pip install stuff
|
|
39
|
-
|
|
39
|
+
shell: bash -el {0}
|
|
40
|
+
run: python -m pip install -e "dartsort/[doc]"
|
|
40
41
|
|
|
41
42
|
- name: Zensical build
|
|
43
|
+
shell: bash -el {0}
|
|
42
44
|
run: pushd dartsort/ && zensical build --clean && popd
|
|
43
45
|
|
|
44
46
|
- name: overwrite site
|
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
# extension: .py
|
|
6
6
|
# format_name: percent
|
|
7
7
|
# format_version: '1.3'
|
|
8
|
-
# jupytext_version: 1.19.
|
|
8
|
+
# jupytext_version: 1.19.6
|
|
9
9
|
# kernelspec:
|
|
10
|
-
# display_name: Python [conda env:
|
|
10
|
+
# display_name: Python [conda env:a]
|
|
11
11
|
# language: python
|
|
12
|
-
# name: conda-env-
|
|
12
|
+
# name: conda-env-a-py
|
|
13
13
|
# ---
|
|
14
14
|
|
|
15
15
|
# %% [markdown]
|
|
@@ -31,18 +31,16 @@
|
|
|
31
31
|
# recording.
|
|
32
32
|
|
|
33
33
|
# %%
|
|
34
|
-
import
|
|
35
|
-
|
|
36
|
-
# %%
|
|
37
|
-
# dandi-related
|
|
38
|
-
from dandi.dandiapi import DandiAPIClient
|
|
39
|
-
import fsspec
|
|
34
|
+
import dataclasses
|
|
40
35
|
|
|
41
|
-
|
|
36
|
+
import matplotlib.pyplot as plt
|
|
42
37
|
import numpy as np
|
|
43
38
|
import spikeinterface.full as si
|
|
39
|
+
from dandi.dandiapi import DandiAPIClient
|
|
40
|
+
|
|
44
41
|
import dartsort
|
|
45
42
|
import dartsort.vis as dartvis
|
|
43
|
+
|
|
46
44
|
dartvis.set_plt_style()
|
|
47
45
|
|
|
48
46
|
# %% [markdown]
|
|
@@ -50,12 +48,10 @@ dartvis.set_plt_style()
|
|
|
50
48
|
|
|
51
49
|
# %%
|
|
52
50
|
# update this for your machine!
|
|
53
|
-
experiment_path = dartsort.ensure_path(
|
|
54
|
-
"~/scratch/ibl-dandi-example/", mkdir=True
|
|
55
|
-
)
|
|
51
|
+
experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
|
|
56
52
|
|
|
57
53
|
# %%
|
|
58
|
-
ds_dir = experiment_path / "dartsort"
|
|
54
|
+
ds_dir = experiment_path / "dartsort-chunked"
|
|
59
55
|
|
|
60
56
|
# %%
|
|
61
57
|
# IBL BWM dandiset, example recording
|
|
@@ -93,7 +89,7 @@ else:
|
|
|
93
89
|
print(rec0)
|
|
94
90
|
|
|
95
91
|
# slice 10 minutes near the end
|
|
96
|
-
#TODO better IBL sampling rate handling
|
|
92
|
+
# TODO better IBL sampling rate handling
|
|
97
93
|
rec = rec0
|
|
98
94
|
end_time = rec.get_end_time()
|
|
99
95
|
start_frame = rec.time_to_sample_index(end_time - 30 * 60)
|
|
@@ -103,36 +99,109 @@ else:
|
|
|
103
99
|
rec._sampling_frequency = 30_000
|
|
104
100
|
|
|
105
101
|
rec = dartsort.preprocess(rec, strategy="ibllike")
|
|
106
|
-
rec = rec.
|
|
102
|
+
rec = rec.save(folder=preprocessing_path, format="binary")
|
|
107
103
|
|
|
108
104
|
# %%
|
|
109
105
|
plt.imshow(
|
|
110
106
|
rec.get_traces(0, 10_000, 11_000).T,
|
|
111
|
-
aspect=
|
|
107
|
+
aspect="auto",
|
|
112
108
|
vmin=-5,
|
|
113
109
|
vmax=5,
|
|
114
|
-
interpolation=
|
|
110
|
+
interpolation="nearest",
|
|
115
111
|
)
|
|
116
|
-
plt.colorbar(shrink=0.5, label=
|
|
117
|
-
plt.xlabel(
|
|
118
|
-
plt.ylabel(
|
|
119
|
-
|
|
112
|
+
plt.colorbar(shrink=0.5, label="standardized voltage")
|
|
113
|
+
plt.xlabel("time (samples")
|
|
114
|
+
plt.ylabel("channels")
|
|
120
115
|
# %% [markdown]
|
|
121
116
|
# # Configuration
|
|
122
117
|
|
|
123
118
|
# %%
|
|
124
119
|
# use defaults, except for the pipeline change
|
|
125
|
-
cfg = dartsort.DARTsortUserConfig(
|
|
120
|
+
cfg = dartsort.DARTsortUserConfig()
|
|
126
121
|
|
|
127
122
|
# %%
|
|
128
123
|
# process into an internal config object for later use
|
|
129
124
|
internal_cfg = dartsort.to_internal_config(cfg)
|
|
130
125
|
|
|
131
126
|
# %% [markdown]
|
|
132
|
-
# #
|
|
127
|
+
# # Initial spike detection and motion tracking model fits
|
|
133
128
|
|
|
134
129
|
# %%
|
|
135
|
-
dartsort.dartsort(rec, ds_dir, cfg=cfg)
|
|
130
|
+
dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_motion_models_only=True))
|
|
131
|
+
|
|
132
|
+
# %% [markdown]
|
|
133
|
+
# # Chunked spike detection for motion tracking
|
|
134
|
+
#
|
|
135
|
+
# While dartsort itself only needs to detect spikes on a subsample of the recording
|
|
136
|
+
# in order to do clustering, motion tracking does require spikes from the whole
|
|
137
|
+
# recording. dartsort uses a cheap but good spike detection and localization pass for
|
|
138
|
+
# this, which is very fast, but users at the thousand hour scale still might want to
|
|
139
|
+
# parallelize it.
|
|
140
|
+
#
|
|
141
|
+
# This runs that cheap spike detection for motion tracking in 1-minute batches for demo
|
|
142
|
+
# purposes.
|
|
143
|
+
#
|
|
144
|
+
# Here, the `chunk_starts_samples` argument slices the recording for us.
|
|
145
|
+
# This is better in this case than using `.frame_slice` or `.time_slice` on the recording,
|
|
146
|
+
# since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
|
|
147
|
+
#
|
|
148
|
+
# If the initial detection already covered the whole recording, motion tracking uses its
|
|
149
|
+
# spikes and this step is skipped.
|
|
150
|
+
|
|
151
|
+
# %%
|
|
152
|
+
subtraction_sorting = dartsort.load(ds_dir / "subtraction.h5")
|
|
153
|
+
need_motion_peaks = cfg.do_motion_estimation and dartsort.motion_needs_peaks(
|
|
154
|
+
internal_cfg, rec, subtraction_sorting
|
|
155
|
+
)
|
|
156
|
+
need_motion_peaks
|
|
157
|
+
|
|
158
|
+
# %%
|
|
159
|
+
chunk_length_samples = internal_cfg.motion_estimation_cfg.threshold_cfg.chunk_length_samples
|
|
160
|
+
batch_length_samples = 60 * chunk_length_samples
|
|
161
|
+
duration_samples = rec.get_num_samples()
|
|
162
|
+
|
|
163
|
+
if need_motion_peaks:
|
|
164
|
+
for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, batch_length_samples)):
|
|
165
|
+
batch_end_samples = min(duration_samples, batch_start_samples + batch_length_samples)
|
|
166
|
+
res = dartsort.detect_for_motion(
|
|
167
|
+
recording=rec,
|
|
168
|
+
output_dir=ds_dir,
|
|
169
|
+
previous_sorting=subtraction_sorting,
|
|
170
|
+
hdf5_filename=f"motionthreshold_batch{batch_ix:05}.h5",
|
|
171
|
+
chunk_starts_samples=np.arange(batch_start_samples, batch_end_samples, chunk_length_samples),
|
|
172
|
+
motion_cfg=internal_cfg.motion_estimation_cfg,
|
|
173
|
+
computation_cfg=internal_cfg.motion_computation_cfg,
|
|
174
|
+
sampling_cfg=internal_cfg.peeler_sampling_cfg,
|
|
175
|
+
waveform_cfg=internal_cfg.waveform_cfg,
|
|
176
|
+
)
|
|
177
|
+
del res
|
|
178
|
+
|
|
179
|
+
# %% [markdown]
|
|
180
|
+
# Combine the output of those detections into one sorting, and check that together the
|
|
181
|
+
# batches covered the recording.
|
|
182
|
+
|
|
183
|
+
# %%
|
|
184
|
+
if need_motion_peaks and not (ds_dir / "motionthreshold.h5").exists():
|
|
185
|
+
motion_chunk_h5s = sorted(ds_dir.glob("motionthreshold_batch*.h5"))
|
|
186
|
+
dartsort.virtual_concatenate_sorting_h5s(motion_chunk_h5s, ds_dir / "motionthreshold.h5")
|
|
187
|
+
motion_sorting = dartsort.load(ds_dir / "motionthreshold.h5")
|
|
188
|
+
assert np.array_equal(
|
|
189
|
+
motion_sorting.load_dataset("chunk_starts_samples"),
|
|
190
|
+
np.arange(0, duration_samples, chunk_length_samples),
|
|
191
|
+
)
|
|
192
|
+
print(motion_sorting)
|
|
193
|
+
|
|
194
|
+
# %%
|
|
195
|
+
# !ls -lah {ds_dir}
|
|
196
|
+
|
|
197
|
+
# %% [markdown]
|
|
198
|
+
# ^ Notice all of the chunked motion files! But motionthreshold.h5 itself is tiny, because the concatenation is virtual.
|
|
199
|
+
|
|
200
|
+
# %% [markdown]
|
|
201
|
+
# # Resume: estimate motion, cluster, learn templates and the matching step classifier
|
|
202
|
+
|
|
203
|
+
# %%
|
|
204
|
+
dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_matching_models_only=True))
|
|
136
205
|
|
|
137
206
|
# %%
|
|
138
207
|
# !ls {ds_dir}
|
|
@@ -146,11 +215,8 @@ dartsort.dartsort(rec, ds_dir, cfg=cfg)
|
|
|
146
215
|
# %% [markdown]
|
|
147
216
|
# # Chunked template matching
|
|
148
217
|
#
|
|
149
|
-
# This runs template matching in 1-minute batches
|
|
150
|
-
#
|
|
151
|
-
# Here, `chunk_starts_samples` slices the recording for us.
|
|
152
|
-
# This might be better than using `.frame_slice` or `.time_slice` on the recording,
|
|
153
|
-
# since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
|
|
218
|
+
# This runs template matching in 1-minute batches, with the same loop over chunks
|
|
219
|
+
# as the motion tracking's spike detection above.
|
|
154
220
|
#
|
|
155
221
|
|
|
156
222
|
# %%
|
|
@@ -186,7 +252,9 @@ for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, 60 * 3
|
|
|
186
252
|
# which can be turned off for users who don't need them.
|
|
187
253
|
|
|
188
254
|
# %%
|
|
189
|
-
sorting = dartsort.concatenate_sortings(
|
|
255
|
+
sorting = dartsort.concatenate_sortings(
|
|
256
|
+
[dartsort.load(h5) for h5 in sorted(ds_dir.glob("matching1_batch*.h5"))]
|
|
257
|
+
)
|
|
190
258
|
|
|
191
259
|
# %%
|
|
192
260
|
sorting = dartsort.cluster(
|
|
@@ -199,7 +267,7 @@ sorting = dartsort.cluster(
|
|
|
199
267
|
|
|
200
268
|
# %%
|
|
201
269
|
# if you want to keep this...
|
|
202
|
-
sorting.save(ds_dir /
|
|
270
|
+
sorting.save(ds_dir / "dartsort_sorting.npz")
|
|
203
271
|
|
|
204
272
|
# %% [markdown]
|
|
205
273
|
# # Visualize
|
|
@@ -5,25 +5,26 @@
|
|
|
5
5
|
# extension: .py
|
|
6
6
|
# format_name: percent
|
|
7
7
|
# format_version: '1.3'
|
|
8
|
-
# jupytext_version: 1.19.
|
|
8
|
+
# jupytext_version: 1.19.6
|
|
9
9
|
# kernelspec:
|
|
10
|
-
# display_name: Python [conda env:
|
|
10
|
+
# display_name: Python [conda env:a]
|
|
11
11
|
# language: python
|
|
12
|
-
# name: conda-env-
|
|
12
|
+
# name: conda-env-a-py
|
|
13
13
|
# ---
|
|
14
14
|
|
|
15
15
|
# %%
|
|
16
16
|
import matplotlib.pyplot as plt
|
|
17
17
|
|
|
18
|
+
# %%
|
|
19
|
+
import spikeinterface.full as si
|
|
20
|
+
|
|
18
21
|
# %%
|
|
19
22
|
# dandi-related
|
|
20
23
|
from dandi.dandiapi import DandiAPIClient
|
|
21
|
-
import fsspec
|
|
22
24
|
|
|
23
|
-
# %%
|
|
24
|
-
import spikeinterface.full as si
|
|
25
25
|
import dartsort
|
|
26
26
|
import dartsort.vis as dartvis
|
|
27
|
+
|
|
27
28
|
dartvis.set_plt_style()
|
|
28
29
|
|
|
29
30
|
# %% [markdown]
|
|
@@ -31,9 +32,7 @@ dartvis.set_plt_style()
|
|
|
31
32
|
|
|
32
33
|
# %%
|
|
33
34
|
# update this for your machine!
|
|
34
|
-
experiment_path = dartsort.ensure_path(
|
|
35
|
-
"~/scratch/ibl-dandi-example/", mkdir=True
|
|
36
|
-
)
|
|
35
|
+
experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
|
|
37
36
|
|
|
38
37
|
# %%
|
|
39
38
|
# IBL BWM dandiset, example recording
|
|
@@ -71,7 +70,7 @@ else:
|
|
|
71
70
|
print(rec0)
|
|
72
71
|
|
|
73
72
|
# slice 10 minutes near the end
|
|
74
|
-
#TODO better IBL sampling rate handling
|
|
73
|
+
# TODO better IBL sampling rate handling
|
|
75
74
|
rec = rec0
|
|
76
75
|
end_time = rec.get_end_time()
|
|
77
76
|
start_frame = rec.time_to_sample_index(end_time - 30 * 60)
|
|
@@ -81,20 +80,19 @@ else:
|
|
|
81
80
|
rec._sampling_frequency = 30_000
|
|
82
81
|
|
|
83
82
|
rec = dartsort.preprocess(rec, strategy="ibllike")
|
|
84
|
-
rec = rec.
|
|
83
|
+
rec = rec.save(folder=preprocessing_path, format="binary")
|
|
85
84
|
|
|
86
85
|
# %%
|
|
87
86
|
plt.imshow(
|
|
88
87
|
rec.get_traces(0, 10_000, 11_000).T,
|
|
89
|
-
aspect=
|
|
88
|
+
aspect="auto",
|
|
90
89
|
vmin=-5,
|
|
91
90
|
vmax=5,
|
|
92
|
-
interpolation=
|
|
91
|
+
interpolation="nearest",
|
|
93
92
|
)
|
|
94
|
-
plt.colorbar(shrink=0.5, label=
|
|
95
|
-
plt.xlabel(
|
|
96
|
-
plt.ylabel(
|
|
97
|
-
|
|
93
|
+
plt.colorbar(shrink=0.5, label="standardized voltage")
|
|
94
|
+
plt.xlabel("time (samples")
|
|
95
|
+
plt.ylabel("channels")
|
|
98
96
|
# %% [markdown]
|
|
99
97
|
# ## Run *dartsort*
|
|
100
98
|
|
|
@@ -106,6 +104,6 @@ ds_res = dartsort.dartsort(rec, ds_dir)
|
|
|
106
104
|
# # Visualize
|
|
107
105
|
|
|
108
106
|
# %%
|
|
109
|
-
dartvis.scatter_spike_features(sorting=ds_res["sorting"])
|
|
107
|
+
dartvis.scatter_spike_features(sorting=ds_res["sorting"])
|
|
110
108
|
|
|
111
109
|
# %%
|
|
@@ -63,6 +63,7 @@ from .util.data_util import (
|
|
|
63
63
|
sorting_from_spikeinterface,
|
|
64
64
|
try_get_denoising_pipeline,
|
|
65
65
|
try_get_model_dir,
|
|
66
|
+
virtual_concatenate_sorting_h5s,
|
|
66
67
|
)
|
|
67
68
|
from .util.drift_util import registered_geometry
|
|
68
69
|
from .util.internal_config import *
|
|
@@ -80,6 +81,7 @@ from .util.logging_util import (
|
|
|
80
81
|
get_logger,
|
|
81
82
|
set_log_level,
|
|
82
83
|
)
|
|
84
|
+
from .util.main_util import motion_needs_peaks
|
|
83
85
|
from .util.motion import (
|
|
84
86
|
MotionInfo,
|
|
85
87
|
detect_for_motion,
|
|
@@ -243,7 +243,7 @@ class StableWaveformFeatures:
|
|
|
243
243
|
# n, nc_extract
|
|
244
244
|
channels: Tensor
|
|
245
245
|
# n, rank, nc_extract
|
|
246
|
-
features: Tensor
|
|
246
|
+
features: Tensor | np.ndarray
|
|
247
247
|
neighborhoods: SpikeNeighborhoods
|
|
248
248
|
erp: StableFeaturesInterpolator
|
|
249
249
|
|
|
@@ -292,6 +292,9 @@ class StableWaveformFeatures:
|
|
|
292
292
|
trim_to_rank=clustering_features_cfg.feature_rank,
|
|
293
293
|
params=clustering_features_cfg.interp_params.normalize(),
|
|
294
294
|
device=computation_cfg.actual_device(),
|
|
295
|
+
store_on_disk=computation_cfg.low_memory,
|
|
296
|
+
tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
|
|
297
|
+
nan_to_num=True,
|
|
295
298
|
)
|
|
296
299
|
else:
|
|
297
300
|
assert sorting.parent_h5_path is not None
|
|
@@ -308,13 +311,18 @@ class StableWaveformFeatures:
|
|
|
308
311
|
trim_to_rank=clustering_features_cfg.feature_rank,
|
|
309
312
|
params=clustering_features_cfg.interp_params.normalize(),
|
|
310
313
|
device=computation_cfg.actual_device(),
|
|
314
|
+
store_on_disk=computation_cfg.low_memory,
|
|
315
|
+
tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
|
|
316
|
+
nan_to_num=True,
|
|
311
317
|
)
|
|
312
318
|
|
|
313
|
-
logger.dartsortdebug(
|
|
319
|
+
logger.dartsortdebug(
|
|
320
|
+
f"StableWaveformFeatures {features.shape=} {computation_cfg.low_memory=}."
|
|
321
|
+
)
|
|
314
322
|
|
|
315
323
|
return cls(
|
|
316
324
|
channels=channels,
|
|
317
|
-
features=
|
|
325
|
+
features=features,
|
|
318
326
|
neighborhoods=spike_neighborhoods,
|
|
319
327
|
erp=erp,
|
|
320
328
|
)
|
|
@@ -323,6 +331,17 @@ class StableWaveformFeatures:
|
|
|
323
331
|
# -- helpers
|
|
324
332
|
|
|
325
333
|
|
|
334
|
+
def _read_ix_maybe_mmap(x: Tensor | np.ndarray, ix: Tensor | np.ndarray | slice) -> Tensor:
|
|
335
|
+
if isinstance(x, Tensor):
|
|
336
|
+
return x[ix]
|
|
337
|
+
if isinstance(ix, slice):
|
|
338
|
+
return torch.from_numpy(np.array(x[ix]))
|
|
339
|
+
if isinstance(ix, Tensor):
|
|
340
|
+
ix = ix.numpy(force=True)
|
|
341
|
+
assert ix.ndim == 1
|
|
342
|
+
return torch.from_numpy(x[ix])
|
|
343
|
+
|
|
344
|
+
|
|
326
345
|
def _interpolate_main_channel_features(
|
|
327
346
|
*,
|
|
328
347
|
sorting: DARTsortSorting,
|
|
@@ -109,7 +109,7 @@ from ..util.spiketorch import (
|
|
|
109
109
|
)
|
|
110
110
|
from ..util.torch_util import BModule, cleanup_and_log_gpu_usage, torch_compiler
|
|
111
111
|
from .cluster_util import decrumb, linkage, maximal_leaf_groups, violation_statistics
|
|
112
|
-
from .clustering_features import StableWaveformFeatures
|
|
112
|
+
from .clustering_features import StableWaveformFeatures, _read_ix_maybe_mmap
|
|
113
113
|
from .kmeans import _one_gumbel_nolog, batched_kmeans, kmeans
|
|
114
114
|
|
|
115
115
|
if TYPE_CHECKING:
|
|
@@ -1568,7 +1568,7 @@ class StreamingSpikeData(BatchedSpikeData):
|
|
|
1568
1568
|
*,
|
|
1569
1569
|
n_candidates: int,
|
|
1570
1570
|
max_n_candidates: int,
|
|
1571
|
-
x: Tensor,
|
|
1571
|
+
x: Tensor | np.ndarray,
|
|
1572
1572
|
duties: Tensor | None,
|
|
1573
1573
|
neighborhoods: SpikeNeighborhoods,
|
|
1574
1574
|
device: torch.device,
|
|
@@ -1660,7 +1660,7 @@ class StreamingSpikeData(BatchedSpikeData):
|
|
|
1660
1660
|
duties = None
|
|
1661
1661
|
else:
|
|
1662
1662
|
duties = self.duties[spike_indices].to(self.device)
|
|
1663
|
-
x = self.x
|
|
1663
|
+
x = _read_ix_maybe_mmap(self.x, spike_indices).to(self.device)
|
|
1664
1664
|
wx, noise_loglik = _whiten_and_noise_score_batch(
|
|
1665
1665
|
x=x, neighb_ids=neighb_ids, neighb_cov=self.neighb_cov
|
|
1666
1666
|
)
|
|
@@ -1753,7 +1753,12 @@ class TruncatedSpikeData(BatchedSpikeData):
|
|
|
1753
1753
|
assert self.storage_device in (device, torch.device("cpu"))
|
|
1754
1754
|
for f in (xt, whitenedx, CmoCooinvx, noise_logliks):
|
|
1755
1755
|
assert f is None or f.device == self.storage_device
|
|
1756
|
-
assert
|
|
1756
|
+
assert (
|
|
1757
|
+
self.storage_device == device
|
|
1758
|
+
or f is None
|
|
1759
|
+
or not f.numel()
|
|
1760
|
+
or f.is_pinned()
|
|
1761
|
+
)
|
|
1757
1762
|
|
|
1758
1763
|
self.score_only = xt is None
|
|
1759
1764
|
assert self.score_only == (CmoCooinvx is None)
|
|
@@ -2846,7 +2851,8 @@ class TruncatedMixtureModel(BaseMixtureModel):
|
|
|
2846
2851
|
denom = stats.N.clamp_(min=self.eps)
|
|
2847
2852
|
tikh = self.p.prior_pseudocount / denom
|
|
2848
2853
|
U.diagonal(dim1=-2, dim2=-1).add_(tikh[:, None])
|
|
2849
|
-
|
|
2854
|
+
Uchol, _ = torch.linalg.cholesky_ex(U)
|
|
2855
|
+
soln = torch.cholesky_solve(stats.R, Uchol)
|
|
2850
2856
|
self.b.means.copy_(soln[:, -1])
|
|
2851
2857
|
self.b.bases.copy_(soln[:, :-1])
|
|
2852
2858
|
|
|
@@ -4544,6 +4550,7 @@ def get_truncated_datasets(
|
|
|
4544
4550
|
noise: EmbeddedNoise | None = None,
|
|
4545
4551
|
stable_features: StableWaveformFeatures | None = None,
|
|
4546
4552
|
save_kwargs: dict | None = None,
|
|
4553
|
+
computation_cfg: ComputationConfig | None = None,
|
|
4547
4554
|
):
|
|
4548
4555
|
# we assume that the core neighborhoods are exactly the same as extract ones
|
|
4549
4556
|
data = get_full_neighborhood_data(
|
|
@@ -4555,6 +4562,7 @@ def get_truncated_datasets(
|
|
|
4555
4562
|
fit_indices=fit_indices,
|
|
4556
4563
|
stable_features=stable_features,
|
|
4557
4564
|
device=device,
|
|
4565
|
+
computation_cfg=computation_cfg,
|
|
4558
4566
|
)
|
|
4559
4567
|
feature_rank, full_features, full_neighbs, train_ixs, train_neighbs, val_ixs, val_neighbs, prgeom, train_weights, val_weights = data # fmt: skip
|
|
4560
4568
|
del stable_features
|
|
@@ -4601,11 +4609,11 @@ def get_truncated_datasets(
|
|
|
4601
4609
|
huber_quantile=noise_huber_quantile(refinement_cfg),
|
|
4602
4610
|
)
|
|
4603
4611
|
|
|
4612
|
+
train_x = _read_ix_maybe_mmap(full_features, train_ixs)
|
|
4604
4613
|
if refinement_cfg.spike_storage == "device" or not sorting.n_units:
|
|
4605
|
-
train_x =
|
|
4614
|
+
train_x = train_x.to(device=device)
|
|
4606
4615
|
elif refinement_cfg.spike_storage == "cpu":
|
|
4607
|
-
assert
|
|
4608
|
-
train_x = full_features[train_ixs]
|
|
4616
|
+
assert train_x.device.type == "cpu"
|
|
4609
4617
|
else:
|
|
4610
4618
|
panic(refinement_cfg.spike_storage)
|
|
4611
4619
|
|
|
@@ -4712,11 +4720,11 @@ def get_truncated_datasets(
|
|
|
4712
4720
|
val_duties = None
|
|
4713
4721
|
else:
|
|
4714
4722
|
val_duties = duties[val_ixs].to(device=device)
|
|
4723
|
+
val_x = _read_ix_maybe_mmap(full_features, val_ixs)
|
|
4715
4724
|
if refinement_cfg.spike_storage == "device":
|
|
4716
|
-
val_x =
|
|
4725
|
+
val_x = val_x.to(device=device)
|
|
4717
4726
|
elif refinement_cfg.spike_storage == "cpu":
|
|
4718
|
-
assert
|
|
4719
|
-
val_x = full_features[val_ixs]
|
|
4727
|
+
assert val_x.device.type == "cpu"
|
|
4720
4728
|
else:
|
|
4721
4729
|
panic(refinement_cfg.spike_storage)
|
|
4722
4730
|
val_data = TruncatedSpikeData.initialize_from_labels(
|
|
@@ -4822,8 +4830,7 @@ def get_full_neighborhood_data(
|
|
|
4822
4830
|
if clustering_features_cfg is not None:
|
|
4823
4831
|
assert 0 < frank <= clustering_features_cfg.feature_rank
|
|
4824
4832
|
assert xfull.shape[0] == full_neighborhoods.b.neighborhood_ids.shape[0]
|
|
4825
|
-
xfull = xfull.
|
|
4826
|
-
xfull = xfull.nan_to_num_()
|
|
4833
|
+
xfull = xfull.reshape(len(xfull), -1)
|
|
4827
4834
|
|
|
4828
4835
|
full_neighborhoods = full_neighborhoods.to(device=device)
|
|
4829
4836
|
|
|
@@ -5008,6 +5015,7 @@ def instantiate_and_bootstrap_tmm(
|
|
|
5008
5015
|
device=device,
|
|
5009
5016
|
rg=rg,
|
|
5010
5017
|
save_kwargs=save_kwargs,
|
|
5018
|
+
computation_cfg=computation_cfg,
|
|
5011
5019
|
)
|
|
5012
5020
|
)
|
|
5013
5021
|
|
|
@@ -5153,7 +5161,7 @@ class TruncatedKmeanspp:
|
|
|
5153
5161
|
def fit_predict(
|
|
5154
5162
|
cls,
|
|
5155
5163
|
*,
|
|
5156
|
-
x: Tensor,
|
|
5164
|
+
x: Tensor | np.ndarray,
|
|
5157
5165
|
train_x: Tensor,
|
|
5158
5166
|
neighborhoods: SpikeNeighborhoods,
|
|
5159
5167
|
neighb_cov: NeighborhoodCovariance,
|
|
@@ -5235,42 +5243,52 @@ class TruncatedKmeanspp:
|
|
|
5235
5243
|
def label(
|
|
5236
5244
|
self,
|
|
5237
5245
|
*,
|
|
5238
|
-
x: Tensor,
|
|
5246
|
+
x: Tensor | np.ndarray,
|
|
5239
5247
|
neighborhoods: SpikeNeighborhoods,
|
|
5240
5248
|
neighb_cov: NeighborhoodCovariance,
|
|
5241
5249
|
centroid_ixs: Tensor,
|
|
5242
5250
|
batch_size: int,
|
|
5251
|
+
chunk_size: int = 2**20,
|
|
5243
5252
|
) -> Tensor:
|
|
5244
5253
|
"""In batches, label each x by its best centroid"""
|
|
5245
5254
|
max_nc_obs = self.neighborhoods.shape[1]
|
|
5246
5255
|
device = self.X.device
|
|
5256
|
+
n_neighbs = neighborhoods.n_neighborhoods
|
|
5247
5257
|
neighb_ids = neighborhoods.b.neighborhood_ids
|
|
5248
5258
|
cent_visible = self.visible[:, self.Xneighbixs[centroid_ixs]]
|
|
5249
5259
|
|
|
5250
|
-
|
|
5251
|
-
|
|
5252
|
-
indptr_list = indptr.tolist()
|
|
5253
|
-
for ni in range(neighborhoods.n_neighborhoods):
|
|
5254
|
-
members = order[indptr_list[ni] : indptr_list[ni + 1]]
|
|
5255
|
-
if not members.numel():
|
|
5256
|
-
continue
|
|
5257
|
-
|
|
5260
|
+
neighb_centers: dict[int, tuple[Tensor, NeighborhoodCenters]] = {}
|
|
5261
|
+
for ni in range(n_neighbs):
|
|
5258
5262
|
(cand,) = cent_visible[ni].nonzero(as_tuple=True)
|
|
5259
5263
|
if not cand.numel():
|
|
5260
5264
|
# kmeans++ ran out of components before covering this neighborhood
|
|
5261
5265
|
continue
|
|
5262
|
-
|
|
5266
|
+
neighb_centers[ni] = cand, self.centers_in_neighborhood(
|
|
5267
|
+
ni, centroid_ixs[cand]
|
|
5268
|
+
)
|
|
5263
5269
|
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5267
|
-
|
|
5268
|
-
|
|
5269
|
-
|
|
5270
|
-
|
|
5271
|
-
|
|
5272
|
-
|
|
5273
|
-
|
|
5270
|
+
full_labels = torch.full(x.shape[:1], -1, dtype=torch.long, device=device)
|
|
5271
|
+
for i0 in range(0, x.shape[0], chunk_size):
|
|
5272
|
+
i1 = min(x.shape[0], i0 + chunk_size)
|
|
5273
|
+
x_chunk = _read_ix_maybe_mmap(x, slice(i0, i1))
|
|
5274
|
+
chunk_neighb_ids = neighb_ids[i0:i1]
|
|
5275
|
+
order, indptr, _ = sort_to_csr(chunk_neighb_ids, n_neighbs)
|
|
5276
|
+
indptr_list = indptr.tolist()
|
|
5277
|
+
for ni, (cand, centers) in neighb_centers.items():
|
|
5278
|
+
members = order[indptr_list[ni] : indptr_list[ni + 1]]
|
|
5279
|
+
for bs in range(0, members.numel(), batch_size):
|
|
5280
|
+
bix = members[bs : bs + batch_size]
|
|
5281
|
+
xb = x_chunk[bix.to(x_chunk.device)].to(device)
|
|
5282
|
+
wx, _ = _whiten_and_noise_score_batch(
|
|
5283
|
+
x=xb,
|
|
5284
|
+
neighb_ids=chunk_neighb_ids[bix],
|
|
5285
|
+
neighb_cov=neighb_cov,
|
|
5286
|
+
)
|
|
5287
|
+
xsq = wx.view(bix.shape[0], self.feat_rank, max_nc_obs)
|
|
5288
|
+
xsq = xsq.square().sum(dim=1)
|
|
5289
|
+
d = self.center_dists(wx, xsq, centers)
|
|
5290
|
+
full_labels[i0 + bix] = cand[d.argmin(dim=1)]
|
|
5291
|
+
del x_chunk
|
|
5274
5292
|
|
|
5275
5293
|
n_unlabeled = (full_labels < 0).sum().item()
|
|
5276
5294
|
if n_unlabeled:
|
|
@@ -18,7 +18,7 @@ from ..util.motion import MotionInfo
|
|
|
18
18
|
from ..util.multiprocessing_util import pool_from_cfg
|
|
19
19
|
from ..util.py_util import databag
|
|
20
20
|
from .cluster_util import hierarchical_cluster, reorder_by_depth
|
|
21
|
-
from .clustering_features import StableWaveformFeatures
|
|
21
|
+
from .clustering_features import StableWaveformFeatures, _read_ix_maybe_mmap
|
|
22
22
|
|
|
23
23
|
logger = get_logger(__name__)
|
|
24
24
|
HAVE_KDEPY = importlib.util.find_spec("KDEpy") is not None
|
|
@@ -63,7 +63,7 @@ def pc_merge(
|
|
|
63
63
|
|
|
64
64
|
# make stable features, no need for core features though.
|
|
65
65
|
kept = np.flatnonzero(subset_sorting.labels >= 0)
|
|
66
|
-
x = stable_features.features
|
|
66
|
+
x = _read_ix_maybe_mmap(stable_features.features, kept)
|
|
67
67
|
x = x[:, : refinement_cfg.pc_merge_rank]
|
|
68
68
|
xlabels = torch.from_numpy(subset_sorting.labels[kept]).to(x.device)
|
|
69
69
|
n_reg_chans = motion.rgeom.shape[0]
|
|
@@ -456,7 +456,7 @@ def collision_cleaning_errors(
|
|
|
456
456
|
|
|
457
457
|
# average feature
|
|
458
458
|
kept = np.flatnonzero(subset_sorting.labels >= 0)
|
|
459
|
-
x = stable_features.features
|
|
459
|
+
x = _read_ix_maybe_mmap(stable_features.features, kept)
|
|
460
460
|
xlabels = torch.from_numpy(subset_sorting.labels[kept]).to(x.device)
|
|
461
461
|
n_reg_chans = motion.rgeom.shape[0]
|
|
462
462
|
means, counts = spiketorch.average_by_label(
|
|
@@ -137,6 +137,8 @@ class DARTsortUserConfig:
|
|
|
137
137
|
workdir_follow_symlinks: bool = False
|
|
138
138
|
tmpdir_parent: str | None = None
|
|
139
139
|
"""Control where tmpdirs are created."""
|
|
140
|
+
low_memory: bool = False
|
|
141
|
+
"""Best effort at not using tons of RAM. If you notice memory problems, please get in touch!"""
|
|
140
142
|
save_intermediates: bool = False
|
|
141
143
|
"""Store all spike features from intermediate steps (for debugging)"""
|
|
142
144
|
|
|
@@ -269,6 +271,7 @@ class DARTsortUserConfig:
|
|
|
269
271
|
"""Chunk length in seconds for online motion tracking"""
|
|
270
272
|
|
|
271
273
|
# -- pipeline control parameters for specific use cases
|
|
274
|
+
fit_motion_models_only: bool = False
|
|
272
275
|
fit_matching_models_only: bool = False
|
|
273
276
|
save_full_final_residual: bool = False
|
|
274
277
|
|