DARTsort 0.5.23__tar.gz → 0.5.25__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.23 → dartsort-0.5.25}/PKG-INFO +2 -1
- {dartsort-0.5.23 → dartsort-0.5.25}/README.md +1 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/docs/index.md +1 -0
- dartsort-0.5.25/notebook/fit_and_batch_matching.py +197 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/__init__.py +4 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/clustering/agglomerate.py +370 -490
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/clustering/cluster_util.py +297 -54
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/clustering/clustering.py +105 -29
- dartsort-0.5.25/src/dartsort/clustering/clustering_features.py +651 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/clustering/density.py +0 -2
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/clustering/mixture.py +1134 -290
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/clustering/refine_util.py +292 -48
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/config.py +78 -26
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/detect/detect.py +113 -53
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/evaluate/comparison.py +14 -2
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/evaluate/hybridkit.py +4 -1
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/evaluate/noise_recording_tools.py +7 -4
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/evaluate/simkit.py +1 -1
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/evaluate/simlib.py +6 -4
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/localize/localize_util.py +7 -4
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/main.py +66 -42
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/__init__.py +3 -4
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/matching.py +1 -4
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/matching_util/compressed_upsampled.py +1 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/peel_base.py +157 -80
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/peel_lib.py +9 -370
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/subtract.py +188 -51
- dartsort-0.5.25/src/dartsort/peel/subtract_util.py +1169 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/threshold.py +3 -3
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/templates/get_templates.py +4 -5
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/templates/postprocess_util.py +2 -120
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/templates/templates.py +6 -6
- dartsort-0.5.25/src/dartsort/transform/__init__.py +8 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/_multichan_denoiser_kit.py +17 -11
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/all_transformers.py +5 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/amortized_localization.py +191 -98
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/decollider.py +349 -63
- dartsort-0.5.25/src/dartsort/transform/matched_filter_net.py +282 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/matching_denoiser.py +1 -1
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/mixture_classifier.py +6 -1
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/pipeline.py +151 -37
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/single_channel_denoiser.py +8 -11
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/supervised_denoiser.py +4 -1
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/temporal_pca.py +77 -51
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/transform_base.py +37 -5
- dartsort-0.5.25/src/dartsort/transform/vq_matched_filter.py +157 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/data_util.py +335 -200
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/internal_config.py +557 -337
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/interpolation_util.py +4 -3
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/main_util.py +67 -49
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/nn_util.py +118 -14
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/peel_util.py +12 -9
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/preprocess_util.py +29 -10
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/py_util.py +2 -2
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/spiketorch.py +69 -6
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/torch_util.py +8 -2
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/waveform_util.py +52 -10
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/analysis_plots.py +64 -20
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/mixture.py +39 -15
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/scatterplots.py +3 -2
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/vismain.py +2 -1
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort.egg-info/PKG-INFO +2 -1
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort.egg-info/SOURCES.txt +5 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/conftest.py +61 -0
- dartsort-0.5.25/tests/test_agglomerate.py +412 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_alignment.py +3 -1
- dartsort-0.5.25/tests/test_cluster_util.py +386 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_clustering.py +14 -3
- dartsort-0.5.25/tests/test_clustering_features.py +87 -0
- dartsort-0.5.25/tests/test_config.py +102 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_dartsort.py +1 -1
- dartsort-0.5.25/tests/test_data_util.py +147 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_matching.py +5 -4
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_mixture_classifier.py +2 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_modeling.py +139 -4
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_spiketorch.py +44 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_subtract.py +219 -14
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_templates.py +11 -13
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_transform.py +1 -1
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_waveform_util.py +7 -3
- dartsort-0.5.23/src/dartsort/clustering/clustering_features.py +0 -362
- dartsort-0.5.23/src/dartsort/transform/__init__.py +0 -4
- dartsort-0.5.23/tests/test_agglomerate.py +0 -35
- dartsort-0.5.23/tests/test_cluster_util.py +0 -209
- dartsort-0.5.23/tests/test_config.py +0 -27
- dartsort-0.5.23/tests/test_data_util.py +0 -74
- {dartsort-0.5.23 → dartsort-0.5.25}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/.gitignore +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/docs/main_api.md +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/environment.yml +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/notebook/.keep +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/notebook/ibl_dandi_example.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/pyproject.toml +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/setup.cfg +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/__main__.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/clustering/kmeans.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/evaluate/analysis.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/motion.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/noise_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_density.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_detect.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_hybridkit.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_linalg.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_simkit.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_threshold.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/tests/test_util.py +0 -0
- {dartsort-0.5.23 → dartsort-0.5.25}/zensical.toml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dartsort
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.25
|
|
4
4
|
Summary: DARTsort
|
|
5
5
|
Author-email: Charlie Windolf <ciw2048@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/cwindolf/dartsort
|
|
@@ -48,6 +48,7 @@ Requires-Dist: matplotlib-venn; extra == "vis"
|
|
|
48
48
|
[](https://coveralls.io/github/cwindolf/dartsort)
|
|
49
49
|
[](https://doi.org/10.5281/zenodo.16943074)
|
|
50
50
|
[](https://pypi.org/p/dartsort)
|
|
51
|
+
[](https://dartsort.zulipchat.org)
|
|
51
52
|
|
|
52
53
|
# dartsort
|
|
53
54
|
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
[](https://coveralls.io/github/cwindolf/dartsort)
|
|
6
6
|
[](https://doi.org/10.5281/zenodo.16943074)
|
|
7
7
|
[](https://pypi.org/p/dartsort)
|
|
8
|
+
[](https://dartsort.zulipchat.org)
|
|
8
9
|
|
|
9
10
|
# dartsort
|
|
10
11
|
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
[](https://coveralls.io/github/cwindolf/dartsort)
|
|
6
6
|
[](https://doi.org/10.5281/zenodo.16943074)
|
|
7
7
|
[](https://pypi.org/p/dartsort)
|
|
8
|
+
[](https://dartsort.zulipchat.org)
|
|
8
9
|
|
|
9
10
|
# dartsort
|
|
10
11
|
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# ---
|
|
2
|
+
# jupyter:
|
|
3
|
+
# jupytext:
|
|
4
|
+
# text_representation:
|
|
5
|
+
# extension: .py
|
|
6
|
+
# format_name: percent
|
|
7
|
+
# format_version: '1.3'
|
|
8
|
+
# jupytext_version: 1.19.5
|
|
9
|
+
# kernelspec:
|
|
10
|
+
# display_name: Python [conda env:dart14]
|
|
11
|
+
# language: python
|
|
12
|
+
# name: conda-env-dart14-py
|
|
13
|
+
# ---
|
|
14
|
+
|
|
15
|
+
# %% [markdown]
|
|
16
|
+
# This example notebook shows how to break dartsort into stages:
|
|
17
|
+
# - Fitting the template matching model
|
|
18
|
+
# - Running template matching
|
|
19
|
+
# - Gathering outputs
|
|
20
|
+
#
|
|
21
|
+
# This pipeline is intended for users with very long recordings who want more control over the matching step.
|
|
22
|
+
#
|
|
23
|
+
# Since the matching step is the only one which passes over the whole recording, long recordings can benefit
|
|
24
|
+
# from additional parallelism there beyond *dartsort*'s built-in single- or multi-GPU parallelism.
|
|
25
|
+
# In this notebook, the matching step is run "chunked", but just in a for loop without parallelism.
|
|
26
|
+
# This keeps it simple here, but shows all of the boilerplate for chunking and gathering results.
|
|
27
|
+
# Users should swap out the for loop with something for their platform (SLURM script, Dask situation, ...).
|
|
28
|
+
#
|
|
29
|
+
#
|
|
30
|
+
# This is a translation of the [ibl_dandi_example](ibl_dandi_example.py) notebook, which works off of a DANDI
|
|
31
|
+
# recording.
|
|
32
|
+
|
|
33
|
+
# %%
|
|
34
|
+
import matplotlib.pyplot as plt
|
|
35
|
+
|
|
36
|
+
# %%
|
|
37
|
+
# dandi-related
|
|
38
|
+
from dandi.dandiapi import DandiAPIClient
|
|
39
|
+
import fsspec
|
|
40
|
+
|
|
41
|
+
# %%
|
|
42
|
+
import numpy as np
|
|
43
|
+
import spikeinterface.full as si
|
|
44
|
+
import dartsort
|
|
45
|
+
import dartsort.vis as dartvis
|
|
46
|
+
dartvis.set_plt_style()
|
|
47
|
+
|
|
48
|
+
# %% [markdown]
|
|
49
|
+
# ### Paths / config
|
|
50
|
+
|
|
51
|
+
# %%
|
|
52
|
+
# update this for your machine!
|
|
53
|
+
experiment_path = dartsort.ensure_path(
|
|
54
|
+
"~/scratch/ibl-dandi-example/", mkdir=True
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
# %%
|
|
58
|
+
ds_dir = experiment_path / "dartsort"
|
|
59
|
+
|
|
60
|
+
# %%
|
|
61
|
+
# IBL BWM dandiset, example recording
|
|
62
|
+
dandiset_id = "000409"
|
|
63
|
+
asset_path = "sub-CSH-ZAD-026/sub-CSH-ZAD-026_ses-15763234-d21e-491f-a01b-1238eb96d389_desc-raw_ecephys.nwb"
|
|
64
|
+
|
|
65
|
+
# %% [markdown]
|
|
66
|
+
# ### Preprocess and save a 10 minute snippet of data
|
|
67
|
+
|
|
68
|
+
# %% [markdown]
|
|
69
|
+
# Pre-saving the preprocessing like this is optional. You can also set the
|
|
70
|
+
# preprocessing flag of the `DARTsortUserConfig()` to the same strategy (`"ibllike"`)
|
|
71
|
+
# or one of the others. In that case, you can also set `copy_recording_to_tmpdir` to
|
|
72
|
+
# reduce preprocessing overhead. Here, I'm pre-saving it just to visualize and in
|
|
73
|
+
# case the user wants to experiment with the cached preprocessed recording.
|
|
74
|
+
|
|
75
|
+
# %%
|
|
76
|
+
preprocessing_path = experiment_path / "preprocessed_recording"
|
|
77
|
+
if preprocessing_path.exists():
|
|
78
|
+
rec = si.read_binary_folder(preprocessing_path)
|
|
79
|
+
else:
|
|
80
|
+
si.set_global_job_kwargs(n_jobs=8, pool_engine="thread")
|
|
81
|
+
|
|
82
|
+
# stream from DANDI with SpikeInterface
|
|
83
|
+
# start by finding the s3 asset URL
|
|
84
|
+
with DandiAPIClient() as client:
|
|
85
|
+
asset = client.get_dandiset(dandiset_id, "draft").get_asset_by_path(asset_path)
|
|
86
|
+
s3_url = asset.get_content_url(follow_redirects=1, strip_query=True)
|
|
87
|
+
rec0 = si.read_nwb_recording(
|
|
88
|
+
s3_url,
|
|
89
|
+
stream_mode="fsspec",
|
|
90
|
+
electrical_series_path="acquisition/ElectricalSeriesProbe00AP",
|
|
91
|
+
)
|
|
92
|
+
print("Original recording:")
|
|
93
|
+
print(rec0)
|
|
94
|
+
|
|
95
|
+
# slice 10 minutes near the end
|
|
96
|
+
#TODO better IBL sampling rate handling
|
|
97
|
+
rec = rec0
|
|
98
|
+
end_time = rec.get_end_time()
|
|
99
|
+
start_frame = rec.time_to_sample_index(end_time - 30 * 60)
|
|
100
|
+
end_frame = start_frame + 10 * 60 * 30_000
|
|
101
|
+
rec = rec.frame_slice(start_frame, end_frame)
|
|
102
|
+
rec.reset_times()
|
|
103
|
+
rec._sampling_frequency = 30_000
|
|
104
|
+
|
|
105
|
+
rec = dartsort.preprocess(rec, strategy="ibllike")
|
|
106
|
+
rec = rec.save_to_folder(preprocessing_path)
|
|
107
|
+
|
|
108
|
+
# %%
|
|
109
|
+
plt.imshow(
|
|
110
|
+
rec.get_traces(0, 10_000, 11_000).T,
|
|
111
|
+
aspect='auto',
|
|
112
|
+
vmin=-5,
|
|
113
|
+
vmax=5,
|
|
114
|
+
interpolation='nearest',
|
|
115
|
+
)
|
|
116
|
+
plt.colorbar(shrink=0.5, label='standardized voltage')
|
|
117
|
+
plt.xlabel('time (samples')
|
|
118
|
+
plt.ylabel('channels');
|
|
119
|
+
|
|
120
|
+
# %% [markdown]
|
|
121
|
+
# # Fit models
|
|
122
|
+
|
|
123
|
+
# %%
|
|
124
|
+
dartsort.dartsort(rec, ds_dir, cfg=dartsort.DARTsortUserConfig(fit_matching_models_only=True))
|
|
125
|
+
|
|
126
|
+
# %%
|
|
127
|
+
# !ls {ds_dir}
|
|
128
|
+
|
|
129
|
+
# %%
|
|
130
|
+
# !ls {ds_dir}/matching1_models
|
|
131
|
+
|
|
132
|
+
# %% [markdown]
|
|
133
|
+
# ^ This was our goal: get a motion estimate, template data, and the featurizers and classifiers that template matching needs.
|
|
134
|
+
|
|
135
|
+
# %% [markdown]
|
|
136
|
+
# # Chunked template matching
|
|
137
|
+
#
|
|
138
|
+
# This runs template matching in 1-minute batches.
|
|
139
|
+
#
|
|
140
|
+
# Here, `chunk_starts_samples` slices the recording for us.
|
|
141
|
+
# This might be better than using `.frame_slice` or `.time_slice` on the recording,
|
|
142
|
+
# since this way the sample indices (spike times) are relative to the full recording without extra bookkeeping.
|
|
143
|
+
#
|
|
144
|
+
|
|
145
|
+
# %%
|
|
146
|
+
duration_samples = rec.get_num_samples()
|
|
147
|
+
for batch_ix, batch_start_samples in enumerate(range(0, duration_samples, 60 * 30_000)):
|
|
148
|
+
batch_end_samples = min(duration_samples, batch_start_samples + 60 * 30_000)
|
|
149
|
+
res = dartsort.match(
|
|
150
|
+
recording=rec,
|
|
151
|
+
output_dir=ds_dir,
|
|
152
|
+
model_subdir="matching1_models",
|
|
153
|
+
hdf5_filename=f"matching1_batch{batch_ix:05}.h5",
|
|
154
|
+
chunk_starts_samples=np.arange(batch_start_samples, batch_end_samples, 30_000),
|
|
155
|
+
load_simple_features=False, # ignoring the output, so don't load up extra stuff
|
|
156
|
+
featurization_cfg=dartsort.default_matching_streaming_classifier_cfg,
|
|
157
|
+
motion=dartsort.try_load_motion_info(ds_dir),
|
|
158
|
+
)
|
|
159
|
+
del res
|
|
160
|
+
|
|
161
|
+
# %% [markdown]
|
|
162
|
+
# # Gather results and postprocess
|
|
163
|
+
#
|
|
164
|
+
# In this section, I'm grabbing all of the matching outputs and concatenating them.
|
|
165
|
+
# Then, I'm running *dartsort*'s standard postprocessing: a final merge step and a spike-train
|
|
166
|
+
# cleaning step (duplicate detection removal).
|
|
167
|
+
#
|
|
168
|
+
# Users can change this. If you don't want to merge, remove that config from the list. In that case, you could
|
|
169
|
+
# run the cleaning step on each chunk before concatenating: the merge needs to agree across the batched matching
|
|
170
|
+
# output, but the output of cleaning is the same when run batchwise or on the full result.
|
|
171
|
+
#
|
|
172
|
+
# Here, extra features (amplitudes, localizations) are loaded by the default behavior of `dartsort.load`,
|
|
173
|
+
# which can be turned off for users who don't need them.
|
|
174
|
+
|
|
175
|
+
# %%
|
|
176
|
+
sorting = dartsort.concatenate_sortings([dartsort.load(h5) for h5 in sorted(ds_dir.glob("matching1_batch*.h5"))])
|
|
177
|
+
|
|
178
|
+
# %%
|
|
179
|
+
sorting = dartsort.cluster(
|
|
180
|
+
rec,
|
|
181
|
+
sorting,
|
|
182
|
+
motion=dartsort.try_load_motion_info(ds_dir),
|
|
183
|
+
clustering_cfg=None,
|
|
184
|
+
refinement_cfgs=[dartsort.default_agglomerate_cfg, dartsort.default_clean_cfg],
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
# %%
|
|
188
|
+
# if you want to keep this...
|
|
189
|
+
sorting.save(ds_dir / 'dartsort_sorting.npz')
|
|
190
|
+
|
|
191
|
+
# %% [markdown]
|
|
192
|
+
# # Visualize
|
|
193
|
+
|
|
194
|
+
# %%
|
|
195
|
+
dartvis.scatter_spike_features(sorting=sorting);
|
|
196
|
+
|
|
197
|
+
# %%
|
|
@@ -55,10 +55,14 @@ from .util import (
|
|
|
55
55
|
from .util.data_util import (
|
|
56
56
|
DARTsortSorting,
|
|
57
57
|
check_recording,
|
|
58
|
+
concatenate_sortings,
|
|
58
59
|
get_featurization_pipeline,
|
|
59
60
|
get_tpca,
|
|
60
61
|
load,
|
|
61
62
|
load_stored_tsvd,
|
|
63
|
+
sorting_from_spikeinterface,
|
|
64
|
+
try_get_denoising_pipeline,
|
|
65
|
+
try_get_model_dir,
|
|
62
66
|
)
|
|
63
67
|
from .util.drift_util import registered_geometry
|
|
64
68
|
from .util.internal_config import *
|