DARTsort 0.5.14__tar.gz → 0.5.16__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.14 → dartsort-0.5.16}/.github/workflows/ci.yml +2 -2
- {dartsort-0.5.14 → dartsort-0.5.16}/PKG-INFO +1 -1
- dartsort-0.5.16/notebook/ibl_dandi_example.py +111 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/pyproject.toml +4 -2
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/agglomerate.py +88 -40
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/cluster_util.py +9 -7
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/density.py +1 -1
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/kmeans.py +7 -7
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/mixture.py +158 -77
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/config.py +3 -1
- dartsort-0.5.16/src/dartsort/detect/detect.py +190 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/comparison.py +100 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/simkit.py +24 -17
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching.py +11 -5
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/compressed_upsampled.py +2 -1
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/drifty.py +9 -11
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/matching_base.py +4 -3
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/pairwise.py +1 -3
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/peel_base.py +5 -2
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/peel_lib.py +16 -32
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/shaver.py +1 -8
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/subtract.py +5 -9
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/threshold.py +1 -9
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/decollider.py +6 -5
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/mixture_classifier.py +5 -4
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/pipeline.py +4 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/data_util.py +73 -28
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/internal_config.py +17 -10
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/noise_util.py +2 -2
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/py_util.py +5 -1
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/spiketorch.py +7 -6
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/testing_util/matching_debug_util.py +3 -2
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/vismain.py +31 -17
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/SOURCES.txt +1 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_clustering.py +1 -2
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_dartsort.py +2 -2
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_data_util.py +1 -1
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_detect.py +7 -15
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_modeling.py +13 -11
- dartsort-0.5.14/src/dartsort/detect/detect.py +0 -408
- {dartsort-0.5.14 → dartsort-0.5.16}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/.gitignore +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/README.md +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/docs/index.md +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/docs/main_api.md +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/environment.yml +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/notebook/.keep +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/setup.cfg +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/clustering.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/clustering_features.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/modes.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/clustering/refine_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/analysis.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/main.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/transform_base.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/main_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/motion.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/analysis_plots.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/mixture.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/conftest.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_agglomerate.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_alignment.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_config.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_density.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_linalg.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_matching.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_simkit.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_subtract.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_templates.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_threshold.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_transform.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.14 → dartsort-0.5.16}/zensical.toml +0 -0
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# ---
|
|
2
|
+
# jupyter:
|
|
3
|
+
# jupytext:
|
|
4
|
+
# text_representation:
|
|
5
|
+
# extension: .py
|
|
6
|
+
# format_name: percent
|
|
7
|
+
# format_version: '1.3'
|
|
8
|
+
# jupytext_version: 1.19.3
|
|
9
|
+
# kernelspec:
|
|
10
|
+
# display_name: Python [conda env:dart14]
|
|
11
|
+
# language: python
|
|
12
|
+
# name: conda-env-dart14-py
|
|
13
|
+
# ---
|
|
14
|
+
|
|
15
|
+
# %%
|
|
16
|
+
import matplotlib.pyplot as plt
|
|
17
|
+
|
|
18
|
+
# %%
|
|
19
|
+
# dandi-related
|
|
20
|
+
from dandi.dandiapi import DandiAPIClient
|
|
21
|
+
import fsspec
|
|
22
|
+
|
|
23
|
+
# %%
|
|
24
|
+
import spikeinterface.full as si
|
|
25
|
+
import dartsort
|
|
26
|
+
import dartsort.vis as dartvis
|
|
27
|
+
dartvis.set_plt_style()
|
|
28
|
+
|
|
29
|
+
# %% [markdown]
|
|
30
|
+
# ### Paths / config
|
|
31
|
+
|
|
32
|
+
# %%
|
|
33
|
+
# update this for your machine!
|
|
34
|
+
experiment_path = dartsort.ensure_path(
|
|
35
|
+
"~/scratch/ibl-dandi-example/", mkdir=True
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
# %%
|
|
39
|
+
# IBL BWM dandiset, example recording
|
|
40
|
+
dandiset_id = "000409"
|
|
41
|
+
asset_path = "sub-CSH-ZAD-026/sub-CSH-ZAD-026_ses-15763234-d21e-491f-a01b-1238eb96d389_desc-raw_ecephys.nwb"
|
|
42
|
+
|
|
43
|
+
# %% [markdown]
|
|
44
|
+
# ### Preprocess and save a 10 minute snippet of data
|
|
45
|
+
|
|
46
|
+
# %% [markdown]
|
|
47
|
+
# Pre-saving the preprocessing like this is optional. You can also set the
|
|
48
|
+
# preprocessing flag of the `DARTsortUserConfig()` to the same strategy (`"ibllike"`)
|
|
49
|
+
# or one of the others. In that case, you can also set `copy_recording_to_tmpdir` to
|
|
50
|
+
# reduce preprocessing overhead. Here, I'm pre-saving it just to visualize and in
|
|
51
|
+
# case the user wants to experiment with the cached preprocessed recording.
|
|
52
|
+
|
|
53
|
+
# %%
|
|
54
|
+
preprocessing_path = experiment_path / "preprocessed_recording"
|
|
55
|
+
if preprocessing_path.exists():
|
|
56
|
+
rec = si.read_binary_folder(preprocessing_path)
|
|
57
|
+
else:
|
|
58
|
+
si.set_global_job_kwargs(n_jobs=8, pool_engine="thread")
|
|
59
|
+
|
|
60
|
+
# stream from DANDI with SpikeInterface
|
|
61
|
+
# start by finding the s3 asset URL
|
|
62
|
+
with DandiAPIClient() as client:
|
|
63
|
+
asset = client.get_dandiset(dandiset_id, "draft").get_asset_by_path(asset_path)
|
|
64
|
+
s3_url = asset.get_content_url(follow_redirects=1, strip_query=True)
|
|
65
|
+
rec0 = si.read_nwb_recording(
|
|
66
|
+
s3_url,
|
|
67
|
+
stream_mode="fsspec",
|
|
68
|
+
electrical_series_path="acquisition/ElectricalSeriesProbe00AP",
|
|
69
|
+
)
|
|
70
|
+
print("Original recording:")
|
|
71
|
+
print(rec0)
|
|
72
|
+
|
|
73
|
+
# slice 10 minutes near the end
|
|
74
|
+
#TODO better IBL sampling rate handling
|
|
75
|
+
rec = rec0
|
|
76
|
+
end_time = rec.get_end_time()
|
|
77
|
+
start_frame = rec.time_to_sample_index(end_time - 30 * 60)
|
|
78
|
+
end_frame = start_frame + 10 * 60 * 30_000
|
|
79
|
+
rec = rec.frame_slice(start_frame, end_frame)
|
|
80
|
+
rec.reset_times()
|
|
81
|
+
rec._sampling_frequency = 30_000
|
|
82
|
+
|
|
83
|
+
rec = dartsort.preprocess(rec, strategy="ibllike")
|
|
84
|
+
rec = rec.save_to_folder(preprocessing_path)
|
|
85
|
+
|
|
86
|
+
# %%
|
|
87
|
+
plt.imshow(
|
|
88
|
+
rec.get_traces(0, 10_000, 11_000).T,
|
|
89
|
+
aspect='auto',
|
|
90
|
+
vmin=-5,
|
|
91
|
+
vmax=5,
|
|
92
|
+
interpolation='nearest',
|
|
93
|
+
)
|
|
94
|
+
plt.colorbar(shrink=0.5, label='standardized voltage')
|
|
95
|
+
plt.xlabel('time (samples')
|
|
96
|
+
plt.ylabel('channels');
|
|
97
|
+
|
|
98
|
+
# %% [markdown]
|
|
99
|
+
# ## Run *dartsort*
|
|
100
|
+
|
|
101
|
+
# %%
|
|
102
|
+
ds_dir = experiment_path / "dartsort"
|
|
103
|
+
ds_res = dartsort.dartsort(rec, ds_dir)
|
|
104
|
+
|
|
105
|
+
# %% [markdown]
|
|
106
|
+
# # Visualize
|
|
107
|
+
|
|
108
|
+
# %%
|
|
109
|
+
dartvis.scatter_spike_features(sorting=ds_res["sorting"]);
|
|
110
|
+
|
|
111
|
+
# %%
|
|
@@ -72,8 +72,10 @@ include = 'src\/.*\.pyi?$|tests\/.*\.pyi?$'
|
|
|
72
72
|
|
|
73
73
|
[tool.ruff]
|
|
74
74
|
line-length = 88
|
|
75
|
-
|
|
76
|
-
|
|
75
|
+
|
|
76
|
+
[tool.ruff.lint]
|
|
77
|
+
extend-select = ["E", "F", "I", "N", "B", "SIM", "NPY", "RUF", "PTH", "PD", "FURB"]
|
|
78
|
+
ignore = ["F722", "N812", "SIM102", "SIM108", "E501", "N806", "N803", "N815"]
|
|
77
79
|
|
|
78
80
|
[tool.jupytext]
|
|
79
81
|
"formats" = "ipynb,py:percent"
|
|
@@ -47,8 +47,8 @@ logger = get_logger(__name__)
|
|
|
47
47
|
class Agglomeration:
|
|
48
48
|
agglomerated_sorting: DARTsortSorting
|
|
49
49
|
merge_mapping: np.ndarray
|
|
50
|
-
distances: np.ndarray
|
|
51
|
-
shifts: np.ndarray
|
|
50
|
+
distances: np.ndarray | None
|
|
51
|
+
shifts: np.ndarray | None
|
|
52
52
|
firing_corr: np.ndarray | None
|
|
53
53
|
|
|
54
54
|
|
|
@@ -71,38 +71,69 @@ def agglomerate(
|
|
|
71
71
|
template_merge_cfg = refinement_cfg.template_merge_cfg
|
|
72
72
|
|
|
73
73
|
if template_data is None:
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
recording=recording,
|
|
79
|
-
motion=motion,
|
|
80
|
-
template_data=template_data,
|
|
81
|
-
waveform_cfg=waveform_cfg,
|
|
82
|
-
template_merge_cfg=template_merge_cfg,
|
|
83
|
-
computation_cfg=computation_cfg,
|
|
84
|
-
)
|
|
74
|
+
did_flatten = True
|
|
75
|
+
sorting = sorting.flatten(include_gmm_properties=True)
|
|
76
|
+
else:
|
|
77
|
+
did_flatten = False
|
|
85
78
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
agg_sorting, new_ids = recluster(
|
|
79
|
+
if template_merge_cfg is not None:
|
|
80
|
+
tdist = template_distances(
|
|
89
81
|
sorting=sorting,
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
82
|
+
recording=recording,
|
|
83
|
+
motion=motion,
|
|
84
|
+
template_data=template_data,
|
|
85
|
+
waveform_cfg=waveform_cfg,
|
|
86
|
+
template_merge_cfg=template_merge_cfg,
|
|
87
|
+
computation_cfg=computation_cfg,
|
|
96
88
|
)
|
|
89
|
+
else:
|
|
90
|
+
tdist = None
|
|
91
|
+
|
|
92
|
+
# if not doing any QDA, be done now.
|
|
93
|
+
no_further_glom = (
|
|
94
|
+
(refinement_cfg is None)
|
|
95
|
+
or (template_merge_cfg is None)
|
|
96
|
+
or (not refinement_cfg.qda_threshold)
|
|
97
|
+
)
|
|
98
|
+
if no_further_glom:
|
|
99
|
+
if tdist is not None:
|
|
100
|
+
assert template_merge_cfg is not None
|
|
101
|
+
agg_sorting, new_ids = recluster(
|
|
102
|
+
sorting=sorting,
|
|
103
|
+
unit_ids=tdist.template_data.unit_ids,
|
|
104
|
+
dists=tdist.distances,
|
|
105
|
+
shifts=tdist.shifts,
|
|
106
|
+
unit_snrs=tdist.template_data.snrs_by_channel().max(1),
|
|
107
|
+
threshold=template_merge_cfg.merge_distance_threshold,
|
|
108
|
+
link=template_merge_cfg.linkage,
|
|
109
|
+
)
|
|
110
|
+
elif not did_flatten:
|
|
111
|
+
agg_sorting = sorting.flatten(include_gmm_properties=True)
|
|
112
|
+
new_ids = None
|
|
113
|
+
else:
|
|
114
|
+
agg_sorting = sorting
|
|
115
|
+
new_ids = None
|
|
116
|
+
|
|
117
|
+
if refinement_cfg is not None:
|
|
118
|
+
agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
|
|
119
|
+
|
|
120
|
+
agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
|
|
121
|
+
if new_ids is None:
|
|
122
|
+
new_ids = reorder
|
|
123
|
+
else:
|
|
124
|
+
new_ids = reorder[np.unique(new_ids, return_inverse=True)[1]]
|
|
125
|
+
|
|
97
126
|
return Agglomeration(
|
|
98
127
|
agglomerated_sorting=agg_sorting,
|
|
99
128
|
merge_mapping=new_ids,
|
|
100
|
-
distances=tdist.distances,
|
|
101
|
-
shifts=tdist.shifts,
|
|
129
|
+
distances=None if tdist is None else tdist.distances,
|
|
130
|
+
shifts=None if tdist is None else tdist.shifts,
|
|
102
131
|
firing_corr=None,
|
|
103
132
|
)
|
|
104
133
|
|
|
105
134
|
# tdist tells us the possible merges
|
|
135
|
+
assert tdist is not None
|
|
136
|
+
assert template_merge_cfg is not None
|
|
106
137
|
distance_mask = linkage_mask(
|
|
107
138
|
tdist.distances,
|
|
108
139
|
linkage_method=template_merge_cfg.linkage,
|
|
@@ -119,7 +150,7 @@ def agglomerate(
|
|
|
119
150
|
_oldsum = distance_mask[np.triu_indices_from(distance_mask)].sum()
|
|
120
151
|
fcorr_mask = fcorr <= refinement_cfg.glom_max_firing_corr
|
|
121
152
|
mask = np.logical_and(distance_mask, fcorr_mask)
|
|
122
|
-
np.fill_diagonal(mask, True)
|
|
153
|
+
np.fill_diagonal(mask, val=True)
|
|
123
154
|
_newsum = mask[np.triu_indices_from(mask)].sum()
|
|
124
155
|
logger.dartsortdebug(
|
|
125
156
|
f"Firing corr dropped QDA candidate count from {_oldsum} -> {_newsum}."
|
|
@@ -193,7 +224,7 @@ def agglomerate(
|
|
|
193
224
|
final_mask = np.logical_or(qda_mask, force_mask)
|
|
194
225
|
if si_mask is not None:
|
|
195
226
|
final_mask = np.logical_or(final_mask, si_mask)
|
|
196
|
-
np.fill_diagonal(final_mask, True)
|
|
227
|
+
np.fill_diagonal(final_mask, val=True)
|
|
197
228
|
final_mask_as_distance = np.logical_not(final_mask).astype(np.float32)
|
|
198
229
|
|
|
199
230
|
agg_sorting, new_ids = recluster(
|
|
@@ -615,7 +646,7 @@ def _qda_job(ij):
|
|
|
615
646
|
try:
|
|
616
647
|
kde = FFTKDE(bw="ISJ").fit(dll)
|
|
617
648
|
except ValueError as e:
|
|
618
|
-
logger.dartsortdebug(f"KDEpy error: {
|
|
649
|
+
logger.dartsortdebug(f"KDEpy error: {e}")
|
|
619
650
|
p.score[i, j] = p.score[j, i] = 0.0
|
|
620
651
|
p.min_ratio[i, j] = p.min_ratio[j, i] = 0.0
|
|
621
652
|
return
|
|
@@ -722,7 +753,7 @@ def count_radial_weights(sorting: DARTsortSorting, motion: MotionInfo, radius: f
|
|
|
722
753
|
vv = row[ii]
|
|
723
754
|
vv = vv / vv.sum()
|
|
724
755
|
|
|
725
|
-
for channel, value in zip(ii, vv):
|
|
756
|
+
for channel, value in zip(ii, vv, strict=True):
|
|
726
757
|
cixs = ci[channel]
|
|
727
758
|
cixs = cixs[cixs < motion.rgeom.shape[0]]
|
|
728
759
|
weights[uu, cixs] += value
|
|
@@ -734,7 +765,7 @@ def count_radial_weights(sorting: DARTsortSorting, motion: MotionInfo, radius: f
|
|
|
734
765
|
|
|
735
766
|
def firing_corr(sorting: DARTsortSorting, dt: float, method="binsqrt"):
|
|
736
767
|
if method != "binsqrt":
|
|
737
|
-
|
|
768
|
+
raise ValueError(f"Unknown {method=}.")
|
|
738
769
|
|
|
739
770
|
tsg = sorting.to_tsgroup()
|
|
740
771
|
fr = tsg.count(bin_size=dt) / dt
|
|
@@ -751,6 +782,8 @@ def combine_gmm_scores(
|
|
|
751
782
|
responsibilities = getattr(sorting, f"{old_prefix}_responsibilities", None)
|
|
752
783
|
logliks = getattr(sorting, f"{old_prefix}_log_liks", None)
|
|
753
784
|
|
|
785
|
+
assert not hasattr(sorting, f"{new_prefix}_responsibilities")
|
|
786
|
+
|
|
754
787
|
havec = candidates is not None
|
|
755
788
|
haver = responsibilities is not None
|
|
756
789
|
havel = logliks is not None
|
|
@@ -780,10 +813,9 @@ def combine_gmm_scores(
|
|
|
780
813
|
assert _maxdiff <= 1e-3, _maxdiff
|
|
781
814
|
assert np.greater_equal(np.isneginf(logliks[:, :-1]), candidates == -1).all()
|
|
782
815
|
if sorting.labels is not None:
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
)
|
|
816
|
+
not_noise = np.flatnonzero(sorting.labels >= 0)
|
|
817
|
+
assert np.array_equal(
|
|
818
|
+
sorting.labels[not_noise], new_ids[candidates[not_noise, 0]]
|
|
787
819
|
)
|
|
788
820
|
|
|
789
821
|
# two steps: first merge, then sort
|
|
@@ -839,20 +871,24 @@ def _combine_loop(
|
|
|
839
871
|
mergedl: np.ndarray,
|
|
840
872
|
):
|
|
841
873
|
for s in numba.prange(cand.shape[0]): # ty: ignore
|
|
842
|
-
|
|
874
|
+
spike_cand = cand[s]
|
|
843
875
|
for j in range(cand.shape[1] - 1):
|
|
844
|
-
|
|
845
|
-
|
|
876
|
+
spike_candj = spike_cand[j]
|
|
877
|
+
|
|
878
|
+
# noise or not a merge
|
|
879
|
+
if spike_candj < 0 or new_id_counts[spike_candj] <= 1:
|
|
846
880
|
continue
|
|
847
881
|
|
|
848
|
-
|
|
849
|
-
|
|
882
|
+
# what later indices are equal to me?
|
|
883
|
+
eq_spike_candj = spike_cand[j + 1 :] == spike_candj
|
|
884
|
+
if eq_spike_candj.sum() < 1:
|
|
850
885
|
continue
|
|
851
886
|
|
|
887
|
+
# loop through and combine liks/resps, and -1 out the cands
|
|
852
888
|
rsum = mergedr[s, j]
|
|
853
889
|
lsum = mergedl[s, j]
|
|
854
890
|
for i, k in enumerate(range(j + 1, cand.shape[1])):
|
|
855
|
-
if not
|
|
891
|
+
if not eq_spike_candj[i]:
|
|
856
892
|
continue
|
|
857
893
|
cand[s, k] = -1
|
|
858
894
|
|
|
@@ -867,6 +903,18 @@ def _combine_loop(
|
|
|
867
903
|
mergedr[s, j] = rsum
|
|
868
904
|
mergedl[s, j] = lsum
|
|
869
905
|
|
|
906
|
+
# vacuum into noise component
|
|
907
|
+
# this is partly to handle stuff that was missed before getting here
|
|
908
|
+
# from flatten, for example
|
|
909
|
+
for j in range(cand.shape[1]):
|
|
910
|
+
if 0 <= spike_cand[j] < new_id_counts.shape[0]:
|
|
911
|
+
continue
|
|
912
|
+
rsj = mergedr[s, j]
|
|
913
|
+
if rsj == 0:
|
|
914
|
+
continue
|
|
915
|
+
mergedr[s, -1] += rsj
|
|
916
|
+
mergedr[s, j] = 0.0
|
|
917
|
+
|
|
870
918
|
|
|
871
919
|
def deduplicate_spikes(
|
|
872
920
|
sorting: DARTsortSorting,
|
|
@@ -1028,7 +1076,7 @@ def coentropy_merge_mask(
|
|
|
1028
1076
|
|
|
1029
1077
|
mask = np.logical_or(c.cov >= coverage_threshold, c.iou >= iou_threshold)
|
|
1030
1078
|
mask = np.logical_and(c.coentropy >= min_coentropy, mask)
|
|
1031
|
-
np.fill_diagonal(mask, True)
|
|
1079
|
+
np.fill_diagonal(mask, val=True)
|
|
1032
1080
|
return mask, c
|
|
1033
1081
|
|
|
1034
1082
|
|
|
@@ -74,7 +74,7 @@ def apply_reclustering(
|
|
|
74
74
|
|
|
75
75
|
# align to best snr unit
|
|
76
76
|
times_updated = sorting.times_samples.copy()
|
|
77
|
-
for
|
|
77
|
+
for orig_labels in clust_inverse.values():
|
|
78
78
|
# we don't need to realign clusters which didn't change
|
|
79
79
|
if len(orig_labels) <= 1:
|
|
80
80
|
continue
|
|
@@ -170,7 +170,9 @@ def linkage_mask(
|
|
|
170
170
|
return mask
|
|
171
171
|
|
|
172
172
|
|
|
173
|
-
def sparsify_labels(
|
|
173
|
+
def sparsify_labels(
|
|
174
|
+
labels: np.ndarray, ids: np.ndarray | None = None
|
|
175
|
+
) -> dict[int, np.ndarray]:
|
|
174
176
|
assert labels.ndim == 1
|
|
175
177
|
if ids is None:
|
|
176
178
|
ids = np.unique(labels)
|
|
@@ -186,7 +188,7 @@ def leafsets(Z, max_distance=np.inf):
|
|
|
186
188
|
n = len(Z) + 1
|
|
187
189
|
leaves = {}
|
|
188
190
|
for i, row in enumerate(Z):
|
|
189
|
-
pa, pb, dist,
|
|
191
|
+
pa, pb, dist, _nab = row
|
|
190
192
|
if dist > max_distance:
|
|
191
193
|
break
|
|
192
194
|
leavesa = leaves.get(pa, [int(pa)])
|
|
@@ -206,7 +208,7 @@ def maximal_leaf_groups(
|
|
|
206
208
|
leaves = {k: set(v) for k, v in leaves.items()}
|
|
207
209
|
group_parents = []
|
|
208
210
|
for i, row in reversed(list(enumerate(Z))):
|
|
209
|
-
|
|
211
|
+
_pa, _pb, _dist, nab = row
|
|
210
212
|
if nab > max_group_size:
|
|
211
213
|
continue
|
|
212
214
|
if n + i not in leaves:
|
|
@@ -278,7 +280,7 @@ def is_largest_set_smaller_than(Z, leaf_descendants, max_size=5):
|
|
|
278
280
|
# at each branch, if #descendents<size, branch gets true and
|
|
279
281
|
# its parents are set to false. that way only one ancestor
|
|
280
282
|
# is true for each leaf.
|
|
281
|
-
for i, (pa, pb,
|
|
283
|
+
for i, (pa, pb, _dist, _nab) in enumerate(Z):
|
|
282
284
|
sz = len(leaf_descendants[n_units + i])
|
|
283
285
|
if sz > max_size:
|
|
284
286
|
continue
|
|
@@ -309,7 +311,7 @@ def combine_distances(
|
|
|
309
311
|
has threshold 1.
|
|
310
312
|
"""
|
|
311
313
|
dists = distances[0] / thresholds[0]
|
|
312
|
-
for dist, thresh in zip(distances[1:], thresholds[1:]):
|
|
314
|
+
for dist, thresh in zip(distances[1:], thresholds[1:], strict=True):
|
|
313
315
|
dists = agg_function(dists, dist / thresh)
|
|
314
316
|
return sym_function(dists, dists.T)
|
|
315
317
|
|
|
@@ -525,7 +527,7 @@ def get_main_channel_pcs(
|
|
|
525
527
|
return features
|
|
526
528
|
|
|
527
529
|
|
|
528
|
-
def decrumb(labels: np.ndarray, min_size: int=5, in_place=False, flatten=True):
|
|
530
|
+
def decrumb(labels: np.ndarray, min_size: int = 5, in_place=False, flatten=True):
|
|
529
531
|
"""Remove small units
|
|
530
532
|
|
|
531
533
|
Parameters
|
|
@@ -85,7 +85,7 @@ def get_smoothed_density(
|
|
|
85
85
|
# figure out what bandwidths we'll be working on
|
|
86
86
|
|
|
87
87
|
if do_ramp := bool(sigma_low):
|
|
88
|
-
min_sigma = min(sigma, sigma_low) # type: ignore
|
|
88
|
+
min_sigma = min(sigma, sigma_low) # type: ignore # ty: ignore[x]
|
|
89
89
|
else:
|
|
90
90
|
min_sigma = sigma
|
|
91
91
|
|
|
@@ -3,7 +3,7 @@ import torch
|
|
|
3
3
|
import torch.nn.functional as F
|
|
4
4
|
from torch import Tensor
|
|
5
5
|
|
|
6
|
-
from
|
|
6
|
+
from ..util.py_util import databag, panic
|
|
7
7
|
|
|
8
8
|
try:
|
|
9
9
|
import cupy # type: ignore # ty: ignore[x]
|
|
@@ -58,7 +58,7 @@ def kmeanspp(
|
|
|
58
58
|
closest = torch.cdist(X, X.mean(0, keepdim=True)).argmax()
|
|
59
59
|
centroid_ixs[0] = closest.item()
|
|
60
60
|
else:
|
|
61
|
-
|
|
61
|
+
panic()
|
|
62
62
|
|
|
63
63
|
if Xnormsq is None:
|
|
64
64
|
Xnormsq = torch.linalg.vector_norm(X, dim=1).square_()
|
|
@@ -227,7 +227,7 @@ def truncated_kmeans(
|
|
|
227
227
|
log_likelihoods = None
|
|
228
228
|
|
|
229
229
|
if show_progress:
|
|
230
|
-
it = progrange(n_iter, desc=f"kmeans
|
|
230
|
+
it = progrange(n_iter, desc=f"kmeans sigma={sigma:0.4f}")
|
|
231
231
|
else:
|
|
232
232
|
it = range(n_iter)
|
|
233
233
|
|
|
@@ -323,7 +323,7 @@ def truncated_kmeans(
|
|
|
323
323
|
break
|
|
324
324
|
prev_sigma = sigma
|
|
325
325
|
if show_progress:
|
|
326
|
-
it.set_description(f"kmeans
|
|
326
|
+
it.set_description(f"kmeans sigma={sigma:0.4f}") # type: ignore
|
|
327
327
|
|
|
328
328
|
return dict(
|
|
329
329
|
centroid_ixs=centroid_ixs,
|
|
@@ -447,7 +447,7 @@ def kmeans(
|
|
|
447
447
|
if weights is not None:
|
|
448
448
|
weights = weights.to(device=X.device)
|
|
449
449
|
with torch.jit.optimized_execution(True):
|
|
450
|
-
for
|
|
450
|
+
for _ in range(n_kmeans_tries):
|
|
451
451
|
aa, ee, cc, dists = kmeans_inner(
|
|
452
452
|
X,
|
|
453
453
|
n_kmeanspp_tries=n_kmeanspp_tries,
|
|
@@ -810,7 +810,7 @@ def truncated_kmeans_from_labels(
|
|
|
810
810
|
distsq_buf = (distsq_buf, torch.zeros_like(distsq_buf))
|
|
811
811
|
|
|
812
812
|
if show_progress:
|
|
813
|
-
it = progrange(n_iter, desc=f"kmeans
|
|
813
|
+
it = progrange(n_iter, desc=f"kmeans sigma={sigma:0.4f}")
|
|
814
814
|
else:
|
|
815
815
|
it = range(n_iter)
|
|
816
816
|
|
|
@@ -891,7 +891,7 @@ def truncated_kmeans_from_labels(
|
|
|
891
891
|
|
|
892
892
|
prev_sigma = sigma
|
|
893
893
|
if show_progress:
|
|
894
|
-
it.set_description(f"kmeans
|
|
894
|
+
it.set_description(f"kmeans sigma={sigma:0.4f}") # type: ignore
|
|
895
895
|
|
|
896
896
|
return KMeansResult(
|
|
897
897
|
labels=labels,
|