DARTsort 0.5.32__tar.gz → 0.5.34__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.32 → dartsort-0.5.34}/.github/workflows/docs.yml +7 -5
- {dartsort-0.5.32 → dartsort-0.5.34}/PKG-INFO +1 -2
- {dartsort-0.5.32 → dartsort-0.5.34}/environment.yml +1 -2
- {dartsort-0.5.32 → dartsort-0.5.34}/notebook/fit_and_batch_matching.py +100 -32
- {dartsort-0.5.32 → dartsort-0.5.34}/notebook/ibl_dandi_example.py +16 -18
- {dartsort-0.5.32 → dartsort-0.5.34}/pyproject.toml +0 -1
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/__init__.py +3 -1
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/clustering_features.py +22 -3
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/mixture.py +358 -514
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/refine_util.py +3 -14
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/config.py +6 -5
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/comparison.py +1 -1
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/hybridkit.py +9 -6
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/simkit.py +0 -3
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/simlib.py +274 -256
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/main.py +12 -6
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/realignment.py +1 -1
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/mixture_classifier.py +62 -111
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/reduction.py +36 -24
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/data_util.py +80 -16
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/drift_util.py +61 -13
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/internal_config.py +8 -12
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/interpolation_util.py +84 -49
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/motion.py +42 -40
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/multiprocessing_util.py +9 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/noise_util.py +127 -675
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/py_util.py +26 -1
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/spiketorch.py +5 -58
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/testing_util/mixture_testing_util.py +13 -22
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/mixture.py +7 -18
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/unit_comparison.py +6 -6
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/vismain.py +2 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/PKG-INFO +1 -2
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/SOURCES.txt +0 -2
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/requires.txt +0 -1
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_drift_util.py +6 -4
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_modeling.py +32 -11
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_simkit.py +2 -1
- dartsort-0.5.32/src/dartsort/util/more_operators.py +0 -274
- dartsort-0.5.32/tests/test_linalg.py +0 -83
- {dartsort-0.5.32 → dartsort-0.5.34}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/.gitignore +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/README.md +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/docs/index.md +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/docs/main_api.md +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/notebook/.keep +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/setup.cfg +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/__main__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/agglomerate.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/cluster_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/clustering.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/kmeans.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/analysis.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/peel_base.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/peel_lib.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/subtract.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/subtract_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/decollider.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/matched_filter_net.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/pipeline.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/transform_base.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/vq_matched_filter.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/main_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/analysis_plots.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/conftest.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_agglomerate.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_alignment.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_clustering.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_clustering_features.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_config.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_data_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_density.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_detect.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_hybridkit.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_matching.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_multiprocessing_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_subtract.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_templates.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_threshold.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_transform.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.32 → dartsort-0.5.34}/zensical.toml +0 -0
|
@@ -29,16 +29,18 @@ jobs:
|
|
|
29
29
|
# this is a "fine-grained token"
|
|
30
30
|
token: ${{ secrets.DARTSORT_GITHUB_IO }}
|
|
31
31
|
|
|
32
|
-
- name: Setup
|
|
33
|
-
uses:
|
|
32
|
+
- name: Setup micromamba
|
|
33
|
+
uses: mamba-org/setup-micromamba@v3
|
|
34
34
|
with:
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
environment-file: dartsort/environment.yml
|
|
36
|
+
environment-name: dartsort-docs
|
|
37
37
|
|
|
38
38
|
- name: pip install stuff
|
|
39
|
-
|
|
39
|
+
shell: bash -el {0}
|
|
40
|
+
run: python -m pip install -e "dartsort/[doc]"
|
|
40
41
|
|
|
41
42
|
- name: Zensical build
|
|
43
|
+
shell: bash -el {0}
|
|
42
44
|
run: pushd dartsort/ && zensical build --clean && popd
|
|
43
45
|
|
|
44
46
|
- name: overwrite site
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dartsort
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.34
|
|
4
4
|
Summary: DARTsort
|
|
5
5
|
Author-email: Charlie Windolf <ciw2048@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/cwindolf/dartsort
|
|
@@ -13,7 +13,6 @@ Description-Content-Type: text/markdown
|
|
|
13
13
|
Requires-Dist: dredge-ephys>=0.4.0
|
|
14
14
|
Requires-Dist: h5py
|
|
15
15
|
Requires-Dist: jaxtyping
|
|
16
|
-
Requires-Dist: linear_operator
|
|
17
16
|
Requires-Dist: numba
|
|
18
17
|
Requires-Dist: numpy>=1.20; python_version < "3.13"
|
|
19
18
|
Requires-Dist: numpy>=2.0.0; python_version >= "3.13"
|
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
# extension: .py
|
|
6
6
|
# format_name: percent
|
|
7
7
|
# format_version: '1.3'
|
|
8
|
-
# jupytext_version: 1.19.
|
|
8
|
+
# jupytext_version: 1.19.6
|
|
9
9
|
# kernelspec:
|
|
10
|
-
# display_name: Python [conda env:
|
|
10
|
+
# display_name: Python [conda env:a]
|
|
11
11
|
# language: python
|
|
12
|
-
# name: conda-env-
|
|
12
|
+
# name: conda-env-a-py
|
|
13
13
|
# ---
|
|
14
14
|
|
|
15
15
|
# %% [markdown]
|
|
@@ -31,18 +31,16 @@
|
|
|
31
31
|
# recording.
|
|
32
32
|
|
|
33
33
|
# %%
|
|
34
|
-
import
|
|
35
|
-
|
|
36
|
-
# %%
|
|
37
|
-
# dandi-related
|
|
38
|
-
from dandi.dandiapi import DandiAPIClient
|
|
39
|
-
import fsspec
|
|
34
|
+
import dataclasses
|
|
40
35
|
|
|
41
|
-
|
|
36
|
+
import matplotlib.pyplot as plt
|
|
42
37
|
import numpy as np
|
|
43
38
|
import spikeinterface.full as si
|
|
39
|
+
from dandi.dandiapi import DandiAPIClient
|
|
40
|
+
|
|
44
41
|
import dartsort
|
|
45
42
|
import dartsort.vis as dartvis
|
|
43
|
+
|
|
46
44
|
dartvis.set_plt_style()
|
|
47
45
|
|
|
48
46
|
# %% [markdown]
|
|
@@ -50,12 +48,10 @@ dartvis.set_plt_style()
|
|
|
50
48
|
|
|
51
49
|
# %%
|
|
52
50
|
# update this for your machine!
|
|
53
|
-
experiment_path = dartsort.ensure_path(
|
|
54
|
-
"~/scratch/ibl-dandi-example/", mkdir=True
|
|
55
|
-
)
|
|
51
|
+
experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
|
|
56
52
|
|
|
57
53
|
# %%
|
|
58
|
-
ds_dir = experiment_path / "dartsort"
|
|
54
|
+
ds_dir = experiment_path / "dartsort-chunked"
|
|
59
55
|
|
|
60
56
|
# %%
|
|
61
57
|
# IBL BWM dandiset, example recording
|
|
@@ -93,7 +89,7 @@ else:
|
|
|
93
89
|
print(rec0)
|
|
94
90
|
|
|
95
91
|
# slice 10 minutes near the end
|
|
96
|
-
#TODO better IBL sampling rate handling
|
|
92
|
+
# TODO better IBL sampling rate handling
|
|
97
93
|
rec = rec0
|
|
98
94
|
end_time = rec.get_end_time()
|
|
99
95
|
start_frame = rec.time_to_sample_index(end_time - 30 * 60)
|
|
@@ -103,36 +99,109 @@ else:
|
|
|
103
99
|
rec._sampling_frequency = 30_000
|
|
104
100
|
|
|
105
101
|
rec = dartsort.preprocess(rec, strategy="ibllike")
|
|
106
|
-
rec = rec.
|
|
102
|
+
rec = rec.save(folder=preprocessing_path, format="binary")
|
|
107
103
|
|
|
108
104
|
# %%
|
|
109
105
|
plt.imshow(
|
|
110
106
|
rec.get_traces(0, 10_000, 11_000).T,
|
|
111
|
-
aspect=
|
|
107
|
+
aspect="auto",
|
|
112
108
|
vmin=-5,
|
|
113
109
|
vmax=5,
|
|
114
|
-
interpolation=
|
|
110
|
+
interpolation="nearest",
|
|
115
111
|
)
|
|
116
|
-
plt.colorbar(shrink=0.5, label=
|
|
117
|
-
plt.xlabel(
|
|
118
|
-
plt.ylabel(
|
|
119
|
-
|
|
112
|
+
plt.colorbar(shrink=0.5, label="standardized voltage")
|
|
113
|
+
plt.xlabel("time (samples")
|
|
114
|
+
plt.ylabel("channels")
|
|
120
115
|
# %% [markdown]
|
|
121
116
|
# # Configuration
|
|
122
117
|
|
|
123
118
|
# %%
|
|
124
119
|
# use defaults, except for the pipeline change
|
|
125
|
-
cfg = dartsort.DARTsortUserConfig(
|
|
120
|
+
cfg = dartsort.DARTsortUserConfig()
|
|
126
121
|
|
|
127
122
|
# %%
|
|
128
123
|
# process into an internal config object for later use
|
|
129
124
|
internal_cfg = dartsort.to_internal_config(cfg)
|
|
130
125
|
|
|
131
126
|
# %% [markdown]
|
|
132
|
-
# #
|
|
127
|
+
# # Initial spike detection and motion tracking model fits
|
|
133
128
|
|
|
134
129
|
# %%
|
|
135
|
-
dartsort.dartsort(rec, ds_dir, cfg=cfg)
|
|
130
|
+
dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_motion_models_only=True))
|
|
131
|
+
|
|
132
|
+
# %% [markdown]
|
|
133
|
+
# # Chunked spike detection for motion tracking
|
|
134
|
+
#
|
|
135
|
+
# While dartsort itself only needs to detect spikes on a subsample of the recording
|
|
136
|
+
# in order to do clustering, motion tracking does require spikes from the whole
|
|
137
|
+
# recording. dartsort uses a cheap but good spike detection and localization pass for
|
|
138
|
+
# this, which is very fast, but users at the thousand hour scale still might want to
|
|
139
|
+
# parallelize it.
|
|
140
|
+
#
|
|
141
|
+
# This runs that cheap spike detection for motion tracking in 1-minute batches for demo
|
|
142
|
+
# purposes.
|
|
143
|
+
#
|
|
144
|
+
# Here, the `chunk_starts_samples` argument slices the recording for us.
|
|
145
|
+
# This is better in this case than using `.frame_slice` or `.time_slice` on the recording,
|
|
146
|
+
# since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
|
|
147
|
+
#
|
|
148
|
+
# If the initial detection already covered the whole recording, motion tracking uses its
|
|
149
|
+
# spikes and this step is skipped.
|
|
150
|
+
|
|
151
|
+
# %%
|
|
152
|
+
subtraction_sorting = dartsort.load(ds_dir / "subtraction.h5")
|
|
153
|
+
need_motion_peaks = cfg.do_motion_estimation and dartsort.motion_needs_peaks(
|
|
154
|
+
internal_cfg, rec, subtraction_sorting
|
|
155
|
+
)
|
|
156
|
+
need_motion_peaks
|
|
157
|
+
|
|
158
|
+
# %%
|
|
159
|
+
chunk_length_samples = internal_cfg.motion_estimation_cfg.threshold_cfg.chunk_length_samples
|
|
160
|
+
batch_length_samples = 60 * chunk_length_samples
|
|
161
|
+
duration_samples = rec.get_num_samples()
|
|
162
|
+
|
|
163
|
+
if need_motion_peaks:
|
|
164
|
+
for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, batch_length_samples)):
|
|
165
|
+
batch_end_samples = min(duration_samples, batch_start_samples + batch_length_samples)
|
|
166
|
+
res = dartsort.detect_for_motion(
|
|
167
|
+
recording=rec,
|
|
168
|
+
output_dir=ds_dir,
|
|
169
|
+
previous_sorting=subtraction_sorting,
|
|
170
|
+
hdf5_filename=f"motionthreshold_batch{batch_ix:05}.h5",
|
|
171
|
+
chunk_starts_samples=np.arange(batch_start_samples, batch_end_samples, chunk_length_samples),
|
|
172
|
+
motion_cfg=internal_cfg.motion_estimation_cfg,
|
|
173
|
+
computation_cfg=internal_cfg.motion_computation_cfg,
|
|
174
|
+
sampling_cfg=internal_cfg.peeler_sampling_cfg,
|
|
175
|
+
waveform_cfg=internal_cfg.waveform_cfg,
|
|
176
|
+
)
|
|
177
|
+
del res
|
|
178
|
+
|
|
179
|
+
# %% [markdown]
|
|
180
|
+
# Combine the output of those detections into one sorting, and check that together the
|
|
181
|
+
# batches covered the recording.
|
|
182
|
+
|
|
183
|
+
# %%
|
|
184
|
+
if need_motion_peaks and not (ds_dir / "motionthreshold.h5").exists():
|
|
185
|
+
motion_chunk_h5s = sorted(ds_dir.glob("motionthreshold_batch*.h5"))
|
|
186
|
+
dartsort.virtual_concatenate_sorting_h5s(motion_chunk_h5s, ds_dir / "motionthreshold.h5")
|
|
187
|
+
motion_sorting = dartsort.load(ds_dir / "motionthreshold.h5")
|
|
188
|
+
assert np.array_equal(
|
|
189
|
+
motion_sorting.load_dataset("chunk_starts_samples"),
|
|
190
|
+
np.arange(0, duration_samples, chunk_length_samples),
|
|
191
|
+
)
|
|
192
|
+
print(motion_sorting)
|
|
193
|
+
|
|
194
|
+
# %%
|
|
195
|
+
# !ls -lah {ds_dir}
|
|
196
|
+
|
|
197
|
+
# %% [markdown]
|
|
198
|
+
# ^ Notice all of the chunked motion files! But motionthreshold.h5 itself is tiny, because the concatenation is virtual.
|
|
199
|
+
|
|
200
|
+
# %% [markdown]
|
|
201
|
+
# # Resume: estimate motion, cluster, learn templates and the matching step classifier
|
|
202
|
+
|
|
203
|
+
# %%
|
|
204
|
+
dartsort.dartsort(rec, ds_dir, cfg=dataclasses.replace(cfg, fit_matching_models_only=True))
|
|
136
205
|
|
|
137
206
|
# %%
|
|
138
207
|
# !ls {ds_dir}
|
|
@@ -146,11 +215,8 @@ dartsort.dartsort(rec, ds_dir, cfg=cfg)
|
|
|
146
215
|
# %% [markdown]
|
|
147
216
|
# # Chunked template matching
|
|
148
217
|
#
|
|
149
|
-
# This runs template matching in 1-minute batches
|
|
150
|
-
#
|
|
151
|
-
# Here, `chunk_starts_samples` slices the recording for us.
|
|
152
|
-
# This might be better than using `.frame_slice` or `.time_slice` on the recording,
|
|
153
|
-
# since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
|
|
218
|
+
# This runs template matching in 1-minute batches, with the same loop over chunks
|
|
219
|
+
# as the motion tracking's spike detection above.
|
|
154
220
|
#
|
|
155
221
|
|
|
156
222
|
# %%
|
|
@@ -186,7 +252,9 @@ for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, 60 * 3
|
|
|
186
252
|
# which can be turned off for users who don't need them.
|
|
187
253
|
|
|
188
254
|
# %%
|
|
189
|
-
sorting = dartsort.concatenate_sortings(
|
|
255
|
+
sorting = dartsort.concatenate_sortings(
|
|
256
|
+
[dartsort.load(h5) for h5 in sorted(ds_dir.glob("matching1_batch*.h5"))]
|
|
257
|
+
)
|
|
190
258
|
|
|
191
259
|
# %%
|
|
192
260
|
sorting = dartsort.cluster(
|
|
@@ -199,7 +267,7 @@ sorting = dartsort.cluster(
|
|
|
199
267
|
|
|
200
268
|
# %%
|
|
201
269
|
# if you want to keep this...
|
|
202
|
-
sorting.save(ds_dir /
|
|
270
|
+
sorting.save(ds_dir / "dartsort_sorting.npz")
|
|
203
271
|
|
|
204
272
|
# %% [markdown]
|
|
205
273
|
# # Visualize
|
|
@@ -5,25 +5,26 @@
|
|
|
5
5
|
# extension: .py
|
|
6
6
|
# format_name: percent
|
|
7
7
|
# format_version: '1.3'
|
|
8
|
-
# jupytext_version: 1.19.
|
|
8
|
+
# jupytext_version: 1.19.6
|
|
9
9
|
# kernelspec:
|
|
10
|
-
# display_name: Python [conda env:
|
|
10
|
+
# display_name: Python [conda env:a]
|
|
11
11
|
# language: python
|
|
12
|
-
# name: conda-env-
|
|
12
|
+
# name: conda-env-a-py
|
|
13
13
|
# ---
|
|
14
14
|
|
|
15
15
|
# %%
|
|
16
16
|
import matplotlib.pyplot as plt
|
|
17
17
|
|
|
18
|
+
# %%
|
|
19
|
+
import spikeinterface.full as si
|
|
20
|
+
|
|
18
21
|
# %%
|
|
19
22
|
# dandi-related
|
|
20
23
|
from dandi.dandiapi import DandiAPIClient
|
|
21
|
-
import fsspec
|
|
22
24
|
|
|
23
|
-
# %%
|
|
24
|
-
import spikeinterface.full as si
|
|
25
25
|
import dartsort
|
|
26
26
|
import dartsort.vis as dartvis
|
|
27
|
+
|
|
27
28
|
dartvis.set_plt_style()
|
|
28
29
|
|
|
29
30
|
# %% [markdown]
|
|
@@ -31,9 +32,7 @@ dartvis.set_plt_style()
|
|
|
31
32
|
|
|
32
33
|
# %%
|
|
33
34
|
# update this for your machine!
|
|
34
|
-
experiment_path = dartsort.ensure_path(
|
|
35
|
-
"~/scratch/ibl-dandi-example/", mkdir=True
|
|
36
|
-
)
|
|
35
|
+
experiment_path = dartsort.ensure_path("~/scratch/ibl-dandi-example/", mkdir=True)
|
|
37
36
|
|
|
38
37
|
# %%
|
|
39
38
|
# IBL BWM dandiset, example recording
|
|
@@ -71,7 +70,7 @@ else:
|
|
|
71
70
|
print(rec0)
|
|
72
71
|
|
|
73
72
|
# slice 10 minutes near the end
|
|
74
|
-
#TODO better IBL sampling rate handling
|
|
73
|
+
# TODO better IBL sampling rate handling
|
|
75
74
|
rec = rec0
|
|
76
75
|
end_time = rec.get_end_time()
|
|
77
76
|
start_frame = rec.time_to_sample_index(end_time - 30 * 60)
|
|
@@ -81,20 +80,19 @@ else:
|
|
|
81
80
|
rec._sampling_frequency = 30_000
|
|
82
81
|
|
|
83
82
|
rec = dartsort.preprocess(rec, strategy="ibllike")
|
|
84
|
-
rec = rec.
|
|
83
|
+
rec = rec.save(folder=preprocessing_path, format="binary")
|
|
85
84
|
|
|
86
85
|
# %%
|
|
87
86
|
plt.imshow(
|
|
88
87
|
rec.get_traces(0, 10_000, 11_000).T,
|
|
89
|
-
aspect=
|
|
88
|
+
aspect="auto",
|
|
90
89
|
vmin=-5,
|
|
91
90
|
vmax=5,
|
|
92
|
-
interpolation=
|
|
91
|
+
interpolation="nearest",
|
|
93
92
|
)
|
|
94
|
-
plt.colorbar(shrink=0.5, label=
|
|
95
|
-
plt.xlabel(
|
|
96
|
-
plt.ylabel(
|
|
97
|
-
|
|
93
|
+
plt.colorbar(shrink=0.5, label="standardized voltage")
|
|
94
|
+
plt.xlabel("time (samples")
|
|
95
|
+
plt.ylabel("channels")
|
|
98
96
|
# %% [markdown]
|
|
99
97
|
# ## Run *dartsort*
|
|
100
98
|
|
|
@@ -106,6 +104,6 @@ ds_res = dartsort.dartsort(rec, ds_dir)
|
|
|
106
104
|
# # Visualize
|
|
107
105
|
|
|
108
106
|
# %%
|
|
109
|
-
dartvis.scatter_spike_features(sorting=ds_res["sorting"])
|
|
107
|
+
dartvis.scatter_spike_features(sorting=ds_res["sorting"])
|
|
110
108
|
|
|
111
109
|
# %%
|
|
@@ -63,6 +63,7 @@ from .util.data_util import (
|
|
|
63
63
|
sorting_from_spikeinterface,
|
|
64
64
|
try_get_denoising_pipeline,
|
|
65
65
|
try_get_model_dir,
|
|
66
|
+
virtual_concatenate_sorting_h5s,
|
|
66
67
|
)
|
|
67
68
|
from .util.drift_util import registered_geometry
|
|
68
69
|
from .util.internal_config import *
|
|
@@ -80,13 +81,14 @@ from .util.logging_util import (
|
|
|
80
81
|
get_logger,
|
|
81
82
|
set_log_level,
|
|
82
83
|
)
|
|
84
|
+
from .util.main_util import motion_needs_peaks
|
|
83
85
|
from .util.motion import (
|
|
84
86
|
MotionInfo,
|
|
85
87
|
detect_for_motion,
|
|
86
88
|
get_motion_info,
|
|
87
89
|
try_load_motion_info,
|
|
88
90
|
)
|
|
89
|
-
from .util.noise_util import
|
|
91
|
+
from .util.noise_util import KroneckerNoise
|
|
90
92
|
from .util.preprocess_util import preprocess
|
|
91
93
|
from .util.py_util import databag, ensure_path
|
|
92
94
|
from .util.waveform_util import full_channel_index, make_channel_index
|
|
@@ -243,7 +243,7 @@ class StableWaveformFeatures:
|
|
|
243
243
|
# n, nc_extract
|
|
244
244
|
channels: Tensor
|
|
245
245
|
# n, rank, nc_extract
|
|
246
|
-
features: Tensor
|
|
246
|
+
features: Tensor | np.ndarray
|
|
247
247
|
neighborhoods: SpikeNeighborhoods
|
|
248
248
|
erp: StableFeaturesInterpolator
|
|
249
249
|
|
|
@@ -292,6 +292,9 @@ class StableWaveformFeatures:
|
|
|
292
292
|
trim_to_rank=clustering_features_cfg.feature_rank,
|
|
293
293
|
params=clustering_features_cfg.interp_params.normalize(),
|
|
294
294
|
device=computation_cfg.actual_device(),
|
|
295
|
+
store_on_disk=computation_cfg.low_memory,
|
|
296
|
+
tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
|
|
297
|
+
nan_to_num=True,
|
|
295
298
|
)
|
|
296
299
|
else:
|
|
297
300
|
assert sorting.parent_h5_path is not None
|
|
@@ -308,13 +311,18 @@ class StableWaveformFeatures:
|
|
|
308
311
|
trim_to_rank=clustering_features_cfg.feature_rank,
|
|
309
312
|
params=clustering_features_cfg.interp_params.normalize(),
|
|
310
313
|
device=computation_cfg.actual_device(),
|
|
314
|
+
store_on_disk=computation_cfg.low_memory,
|
|
315
|
+
tmpdir_parent=computation_cfg.maybe_tmpdir_parent(),
|
|
316
|
+
nan_to_num=True,
|
|
311
317
|
)
|
|
312
318
|
|
|
313
|
-
logger.dartsortdebug(
|
|
319
|
+
logger.dartsortdebug(
|
|
320
|
+
f"StableWaveformFeatures {features.shape=} {computation_cfg.low_memory=}."
|
|
321
|
+
)
|
|
314
322
|
|
|
315
323
|
return cls(
|
|
316
324
|
channels=channels,
|
|
317
|
-
features=
|
|
325
|
+
features=features,
|
|
318
326
|
neighborhoods=spike_neighborhoods,
|
|
319
327
|
erp=erp,
|
|
320
328
|
)
|
|
@@ -323,6 +331,17 @@ class StableWaveformFeatures:
|
|
|
323
331
|
# -- helpers
|
|
324
332
|
|
|
325
333
|
|
|
334
|
+
def _read_ix_maybe_mmap(x: Tensor | np.ndarray, ix: Tensor | np.ndarray | slice) -> Tensor:
|
|
335
|
+
if isinstance(x, Tensor):
|
|
336
|
+
return x[ix]
|
|
337
|
+
if isinstance(ix, slice):
|
|
338
|
+
return torch.from_numpy(np.array(x[ix]))
|
|
339
|
+
if isinstance(ix, Tensor):
|
|
340
|
+
ix = ix.numpy(force=True)
|
|
341
|
+
assert ix.ndim == 1
|
|
342
|
+
return torch.from_numpy(x[ix])
|
|
343
|
+
|
|
344
|
+
|
|
326
345
|
def _interpolate_main_channel_features(
|
|
327
346
|
*,
|
|
328
347
|
sorting: DARTsortSorting,
|