DARTsort 0.5.15__tar.gz → 0.5.17__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.15 → dartsort-0.5.17}/.github/workflows/ci.yml +3 -2
- {dartsort-0.5.15 → dartsort-0.5.17}/PKG-INFO +1 -1
- {dartsort-0.5.15 → dartsort-0.5.17}/pyproject.toml +7 -2
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/__init__.py +0 -1
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/agglomerate.py +60 -30
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/cluster_util.py +9 -7
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/clustering.py +1 -6
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/clustering_features.py +2 -4
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/density.py +16 -12
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/kmeans.py +7 -7
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/merge.py +1 -1
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/mixture.py +145 -74
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/refine_util.py +6 -59
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/config.py +3 -3
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/detect/detect.py +20 -14
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/evaluate/comparison.py +101 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/evaluate/hybrid_util.py +8 -6
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/evaluate/simkit.py +24 -17
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/localize/localize_torch.py +6 -5
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/main.py +11 -9
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/grab.py +6 -4
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/matching.py +13 -5
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/matching_util/compressed_upsampled.py +5 -1
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/matching_util/drifty.py +15 -12
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/matching_util/matching_base.py +7 -4
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/matching_util/pairwise.py +1 -3
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/peel_base.py +27 -28
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/peel_lib.py +19 -17
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/subtract.py +2 -1
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/amortized_localization.py +12 -13
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/decollider.py +7 -6
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/mixture_classifier.py +5 -4
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/pipeline.py +4 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/whiten.py +4 -1
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/data_util.py +73 -28
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/drift_util.py +3 -3
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/internal_config.py +24 -13
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/interpolation_util.py +17 -16
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/motion.py +12 -12
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/noise_util.py +13 -63
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/py_util.py +5 -1
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/spiketorch.py +12 -10
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/testing_util/matching_debug_util.py +13 -8
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -3
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/waveform_util.py +17 -23
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/mixture.py +25 -24
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/vismain.py +37 -21
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_clustering.py +1 -2
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_dartsort.py +10 -5
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_matching.py +14 -11
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_modeling.py +13 -11
- {dartsort-0.5.15 → dartsort-0.5.17}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/.gitignore +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/README.md +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/docs/index.md +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/docs/main_api.md +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/environment.yml +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/notebook/.keep +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/notebook/ibl_dandi_example.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/setup.cfg +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/clustering/modes.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/evaluate/analysis.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/transform/transform_base.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/main_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/analysis_plots.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort.egg-info/SOURCES.txt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/conftest.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_agglomerate.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_alignment.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_config.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_data_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_density.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_detect.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_linalg.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_simkit.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_subtract.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_templates.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_threshold.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_transform.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.15 → dartsort-0.5.17}/zensical.toml +0 -0
|
@@ -6,8 +6,8 @@ on:
|
|
|
6
6
|
- main
|
|
7
7
|
- develop
|
|
8
8
|
pull_request:
|
|
9
|
-
branches:
|
|
10
|
-
-
|
|
9
|
+
branches-ignore:
|
|
10
|
+
- develop
|
|
11
11
|
workflow_dispatch:
|
|
12
12
|
|
|
13
13
|
jobs:
|
|
@@ -46,3 +46,4 @@ jobs:
|
|
|
46
46
|
uses: coverallsapp/github-action@master
|
|
47
47
|
with:
|
|
48
48
|
path-to-lcov: coverage.lcov
|
|
49
|
+
fail-on-error: false
|
|
@@ -72,8 +72,13 @@ include = 'src\/.*\.pyi?$|tests\/.*\.pyi?$'
|
|
|
72
72
|
|
|
73
73
|
[tool.ruff]
|
|
74
74
|
line-length = 88
|
|
75
|
-
|
|
76
|
-
|
|
75
|
+
|
|
76
|
+
[tool.ruff.lint]
|
|
77
|
+
extend-select = ["E", "F", "I", "N", "B", "SIM", "NPY", "RUF", "PTH", "PD", "FURB"]
|
|
78
|
+
ignore = ["F722", "N812", "SIM102", "SIM108", "E501", "N806", "N803", "N815", "RUF005", "SIM300", "C408"]
|
|
79
|
+
|
|
80
|
+
[tool.ruff.lint.per-file-ignores]
|
|
81
|
+
"**/__init__.py" = ["F401", "F403"]
|
|
77
82
|
|
|
78
83
|
[tool.jupytext]
|
|
79
84
|
"formats" = "ipynb,py:percent"
|
|
@@ -47,8 +47,8 @@ logger = get_logger(__name__)
|
|
|
47
47
|
class Agglomeration:
|
|
48
48
|
agglomerated_sorting: DARTsortSorting
|
|
49
49
|
merge_mapping: np.ndarray
|
|
50
|
-
distances: np.ndarray
|
|
51
|
-
shifts: np.ndarray
|
|
50
|
+
distances: np.ndarray | None
|
|
51
|
+
shifts: np.ndarray | None
|
|
52
52
|
firing_corr: np.ndarray | None
|
|
53
53
|
|
|
54
54
|
|
|
@@ -71,38 +71,69 @@ def agglomerate(
|
|
|
71
71
|
template_merge_cfg = refinement_cfg.template_merge_cfg
|
|
72
72
|
|
|
73
73
|
if template_data is None:
|
|
74
|
+
did_flatten = True
|
|
74
75
|
sorting = sorting.flatten(include_gmm_properties=True)
|
|
76
|
+
else:
|
|
77
|
+
did_flatten = False
|
|
75
78
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
recording=recording,
|
|
79
|
-
motion=motion,
|
|
80
|
-
template_data=template_data,
|
|
81
|
-
waveform_cfg=waveform_cfg,
|
|
82
|
-
template_merge_cfg=template_merge_cfg,
|
|
83
|
-
computation_cfg=computation_cfg,
|
|
84
|
-
)
|
|
85
|
-
|
|
86
|
-
# if not doing any QDA, be done now.
|
|
87
|
-
if refinement_cfg is None or not refinement_cfg.qda_threshold:
|
|
88
|
-
agg_sorting, new_ids = recluster(
|
|
79
|
+
if template_merge_cfg is not None:
|
|
80
|
+
tdist = template_distances(
|
|
89
81
|
sorting=sorting,
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
82
|
+
recording=recording,
|
|
83
|
+
motion=motion,
|
|
84
|
+
template_data=template_data,
|
|
85
|
+
waveform_cfg=waveform_cfg,
|
|
86
|
+
template_merge_cfg=template_merge_cfg,
|
|
87
|
+
computation_cfg=computation_cfg,
|
|
96
88
|
)
|
|
89
|
+
else:
|
|
90
|
+
tdist = None
|
|
91
|
+
|
|
92
|
+
# if not doing any QDA, be done now.
|
|
93
|
+
no_further_glom = (
|
|
94
|
+
(refinement_cfg is None)
|
|
95
|
+
or (template_merge_cfg is None)
|
|
96
|
+
or (not refinement_cfg.qda_threshold)
|
|
97
|
+
)
|
|
98
|
+
if no_further_glom:
|
|
99
|
+
if tdist is not None:
|
|
100
|
+
assert template_merge_cfg is not None
|
|
101
|
+
agg_sorting, new_ids = recluster(
|
|
102
|
+
sorting=sorting,
|
|
103
|
+
unit_ids=tdist.template_data.unit_ids,
|
|
104
|
+
dists=tdist.distances,
|
|
105
|
+
shifts=tdist.shifts,
|
|
106
|
+
unit_snrs=tdist.template_data.snrs_by_channel().max(1),
|
|
107
|
+
threshold=template_merge_cfg.merge_distance_threshold,
|
|
108
|
+
link=template_merge_cfg.linkage,
|
|
109
|
+
)
|
|
110
|
+
elif not did_flatten:
|
|
111
|
+
agg_sorting = sorting.flatten(include_gmm_properties=True)
|
|
112
|
+
new_ids = None
|
|
113
|
+
else:
|
|
114
|
+
agg_sorting = sorting
|
|
115
|
+
new_ids = None
|
|
116
|
+
|
|
117
|
+
if refinement_cfg is not None:
|
|
118
|
+
agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
|
|
119
|
+
|
|
120
|
+
agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
|
|
121
|
+
if new_ids is None:
|
|
122
|
+
new_ids = reorder
|
|
123
|
+
else:
|
|
124
|
+
new_ids = reorder[np.unique(new_ids, return_inverse=True)[1]]
|
|
125
|
+
|
|
97
126
|
return Agglomeration(
|
|
98
127
|
agglomerated_sorting=agg_sorting,
|
|
99
128
|
merge_mapping=new_ids,
|
|
100
|
-
distances=tdist.distances,
|
|
101
|
-
shifts=tdist.shifts,
|
|
129
|
+
distances=None if tdist is None else tdist.distances,
|
|
130
|
+
shifts=None if tdist is None else tdist.shifts,
|
|
102
131
|
firing_corr=None,
|
|
103
132
|
)
|
|
104
133
|
|
|
105
134
|
# tdist tells us the possible merges
|
|
135
|
+
assert tdist is not None
|
|
136
|
+
assert template_merge_cfg is not None
|
|
106
137
|
distance_mask = linkage_mask(
|
|
107
138
|
tdist.distances,
|
|
108
139
|
linkage_method=template_merge_cfg.linkage,
|
|
@@ -119,7 +150,7 @@ def agglomerate(
|
|
|
119
150
|
_oldsum = distance_mask[np.triu_indices_from(distance_mask)].sum()
|
|
120
151
|
fcorr_mask = fcorr <= refinement_cfg.glom_max_firing_corr
|
|
121
152
|
mask = np.logical_and(distance_mask, fcorr_mask)
|
|
122
|
-
np.fill_diagonal(mask, True)
|
|
153
|
+
np.fill_diagonal(mask, val=True)
|
|
123
154
|
_newsum = mask[np.triu_indices_from(mask)].sum()
|
|
124
155
|
logger.dartsortdebug(
|
|
125
156
|
f"Firing corr dropped QDA candidate count from {_oldsum} -> {_newsum}."
|
|
@@ -193,7 +224,7 @@ def agglomerate(
|
|
|
193
224
|
final_mask = np.logical_or(qda_mask, force_mask)
|
|
194
225
|
if si_mask is not None:
|
|
195
226
|
final_mask = np.logical_or(final_mask, si_mask)
|
|
196
|
-
np.fill_diagonal(final_mask, True)
|
|
227
|
+
np.fill_diagonal(final_mask, val=True)
|
|
197
228
|
final_mask_as_distance = np.logical_not(final_mask).astype(np.float32)
|
|
198
229
|
|
|
199
230
|
agg_sorting, new_ids = recluster(
|
|
@@ -576,7 +607,6 @@ class QDACtx:
|
|
|
576
607
|
|
|
577
608
|
|
|
578
609
|
def _qda_init(ctx):
|
|
579
|
-
global _qda_context
|
|
580
610
|
_qda_context.ctx = ctx
|
|
581
611
|
|
|
582
612
|
|
|
@@ -615,7 +645,7 @@ def _qda_job(ij):
|
|
|
615
645
|
try:
|
|
616
646
|
kde = FFTKDE(bw="ISJ").fit(dll)
|
|
617
647
|
except ValueError as e:
|
|
618
|
-
logger.dartsortdebug(f"KDEpy error: {
|
|
648
|
+
logger.dartsortdebug(f"KDEpy error: {e}")
|
|
619
649
|
p.score[i, j] = p.score[j, i] = 0.0
|
|
620
650
|
p.min_ratio[i, j] = p.min_ratio[j, i] = 0.0
|
|
621
651
|
return
|
|
@@ -722,7 +752,7 @@ def count_radial_weights(sorting: DARTsortSorting, motion: MotionInfo, radius: f
|
|
|
722
752
|
vv = row[ii]
|
|
723
753
|
vv = vv / vv.sum()
|
|
724
754
|
|
|
725
|
-
for channel, value in zip(ii, vv):
|
|
755
|
+
for channel, value in zip(ii, vv, strict=True):
|
|
726
756
|
cixs = ci[channel]
|
|
727
757
|
cixs = cixs[cixs < motion.rgeom.shape[0]]
|
|
728
758
|
weights[uu, cixs] += value
|
|
@@ -734,7 +764,7 @@ def count_radial_weights(sorting: DARTsortSorting, motion: MotionInfo, radius: f
|
|
|
734
764
|
|
|
735
765
|
def firing_corr(sorting: DARTsortSorting, dt: float, method="binsqrt"):
|
|
736
766
|
if method != "binsqrt":
|
|
737
|
-
|
|
767
|
+
raise ValueError(f"Unknown {method=}.")
|
|
738
768
|
|
|
739
769
|
tsg = sorting.to_tsgroup()
|
|
740
770
|
fr = tsg.count(bin_size=dt) / dt
|
|
@@ -1045,7 +1075,7 @@ def coentropy_merge_mask(
|
|
|
1045
1075
|
|
|
1046
1076
|
mask = np.logical_or(c.cov >= coverage_threshold, c.iou >= iou_threshold)
|
|
1047
1077
|
mask = np.logical_and(c.coentropy >= min_coentropy, mask)
|
|
1048
|
-
np.fill_diagonal(mask, True)
|
|
1078
|
+
np.fill_diagonal(mask, val=True)
|
|
1049
1079
|
return mask, c
|
|
1050
1080
|
|
|
1051
1081
|
|
|
@@ -74,7 +74,7 @@ def apply_reclustering(
|
|
|
74
74
|
|
|
75
75
|
# align to best snr unit
|
|
76
76
|
times_updated = sorting.times_samples.copy()
|
|
77
|
-
for
|
|
77
|
+
for orig_labels in clust_inverse.values():
|
|
78
78
|
# we don't need to realign clusters which didn't change
|
|
79
79
|
if len(orig_labels) <= 1:
|
|
80
80
|
continue
|
|
@@ -170,7 +170,9 @@ def linkage_mask(
|
|
|
170
170
|
return mask
|
|
171
171
|
|
|
172
172
|
|
|
173
|
-
def sparsify_labels(
|
|
173
|
+
def sparsify_labels(
|
|
174
|
+
labels: np.ndarray, ids: np.ndarray | None = None
|
|
175
|
+
) -> dict[int, np.ndarray]:
|
|
174
176
|
assert labels.ndim == 1
|
|
175
177
|
if ids is None:
|
|
176
178
|
ids = np.unique(labels)
|
|
@@ -186,7 +188,7 @@ def leafsets(Z, max_distance=np.inf):
|
|
|
186
188
|
n = len(Z) + 1
|
|
187
189
|
leaves = {}
|
|
188
190
|
for i, row in enumerate(Z):
|
|
189
|
-
pa, pb, dist,
|
|
191
|
+
pa, pb, dist, _nab = row
|
|
190
192
|
if dist > max_distance:
|
|
191
193
|
break
|
|
192
194
|
leavesa = leaves.get(pa, [int(pa)])
|
|
@@ -206,7 +208,7 @@ def maximal_leaf_groups(
|
|
|
206
208
|
leaves = {k: set(v) for k, v in leaves.items()}
|
|
207
209
|
group_parents = []
|
|
208
210
|
for i, row in reversed(list(enumerate(Z))):
|
|
209
|
-
|
|
211
|
+
_pa, _pb, _dist, nab = row
|
|
210
212
|
if nab > max_group_size:
|
|
211
213
|
continue
|
|
212
214
|
if n + i not in leaves:
|
|
@@ -278,7 +280,7 @@ def is_largest_set_smaller_than(Z, leaf_descendants, max_size=5):
|
|
|
278
280
|
# at each branch, if #descendents<size, branch gets true and
|
|
279
281
|
# its parents are set to false. that way only one ancestor
|
|
280
282
|
# is true for each leaf.
|
|
281
|
-
for i, (pa, pb,
|
|
283
|
+
for i, (pa, pb, _dist, _nab) in enumerate(Z):
|
|
282
284
|
sz = len(leaf_descendants[n_units + i])
|
|
283
285
|
if sz > max_size:
|
|
284
286
|
continue
|
|
@@ -309,7 +311,7 @@ def combine_distances(
|
|
|
309
311
|
has threshold 1.
|
|
310
312
|
"""
|
|
311
313
|
dists = distances[0] / thresholds[0]
|
|
312
|
-
for dist, thresh in zip(distances[1:], thresholds[1:]):
|
|
314
|
+
for dist, thresh in zip(distances[1:], thresholds[1:], strict=True):
|
|
313
315
|
dists = agg_function(dists, dist / thresh)
|
|
314
316
|
return sym_function(dists, dists.T)
|
|
315
317
|
|
|
@@ -525,7 +527,7 @@ def get_main_channel_pcs(
|
|
|
525
527
|
return features
|
|
526
528
|
|
|
527
529
|
|
|
528
|
-
def decrumb(labels: np.ndarray, min_size: int=5, in_place=False, flatten=True):
|
|
530
|
+
def decrumb(labels: np.ndarray, min_size: int = 5, in_place=False, flatten=True):
|
|
529
531
|
"""Remove small units
|
|
530
532
|
|
|
531
533
|
Parameters
|
|
@@ -159,12 +159,7 @@ class Clusterer:
|
|
|
159
159
|
mask_ix = np.flatnonzero(features.keep_mask)
|
|
160
160
|
else:
|
|
161
161
|
mask_ix = None
|
|
162
|
-
if self.sampling_cfg is None:
|
|
163
|
-
if mask_ix is not None:
|
|
164
|
-
return True, mask_ix
|
|
165
|
-
else:
|
|
166
|
-
return False, slice(None)
|
|
167
|
-
elif features.n <= self.sampling_cfg.n_waveforms_fit:
|
|
162
|
+
if self.sampling_cfg is None or features.n <= self.sampling_cfg.n_waveforms_fit:
|
|
168
163
|
if mask_ix is not None:
|
|
169
164
|
return True, mask_ix
|
|
170
165
|
else:
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
from typing import Self, cast
|
|
2
|
-
from logging import WARN
|
|
3
2
|
|
|
4
3
|
import h5py
|
|
5
4
|
import numpy as np
|
|
@@ -131,7 +130,6 @@ class SimpleMatrixFeatures:
|
|
|
131
130
|
samp = amp * np.sign(v)
|
|
132
131
|
|
|
133
132
|
if clustering_features_cfg.use_signed_amplitude:
|
|
134
|
-
samp *= clustering_features_cfg.amp_scale
|
|
135
133
|
features.append(clustering_features_cfg.amp_scale * samp[:, None])
|
|
136
134
|
|
|
137
135
|
do_pcs = bool(clustering_features_cfg.n_main_channel_pcs)
|
|
@@ -170,7 +168,7 @@ class SimpleMatrixFeatures:
|
|
|
170
168
|
pcs,
|
|
171
169
|
raise_for_numerics=clustering_features_cfg.raise_for_numerics,
|
|
172
170
|
)
|
|
173
|
-
|
|
171
|
+
_erp, pcs = interpolate_by_chunk(
|
|
174
172
|
mask=mask,
|
|
175
173
|
dataset=pcs,
|
|
176
174
|
geom=motion.geom,
|
|
@@ -190,7 +188,7 @@ class SimpleMatrixFeatures:
|
|
|
190
188
|
else:
|
|
191
189
|
assert sorting.parent_h5_path is not None
|
|
192
190
|
with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
|
|
193
|
-
|
|
191
|
+
_erp, pcs = interpolate_by_chunk(
|
|
194
192
|
mask=mask,
|
|
195
193
|
dataset=h5[clustering_features_cfg.pca_dataset_name],
|
|
196
194
|
geom=motion.geom,
|
|
@@ -105,7 +105,10 @@ def get_smoothed_density(
|
|
|
105
105
|
)
|
|
106
106
|
nbins = np.ceil(dextents / bin_sizes).astype(int)
|
|
107
107
|
nbins = nbins.clip(min_n_bins, max_n_bins)
|
|
108
|
-
bin_edges = [
|
|
108
|
+
bin_edges = [
|
|
109
|
+
np.linspace(e[0], e[1], num=nb + 1)
|
|
110
|
+
for e, nb in zip(extents, nbins, strict=True)
|
|
111
|
+
]
|
|
109
112
|
|
|
110
113
|
# compute histogram and figure out how big the bins actually were
|
|
111
114
|
raw_histogram, bin_edges = np.histogramdd(infeats, bins=bin_edges)
|
|
@@ -178,7 +181,7 @@ def get_smoothed_density_ratio(
|
|
|
178
181
|
max_n_bins=max_n_bins,
|
|
179
182
|
min_n_bins=min_n_bins,
|
|
180
183
|
)
|
|
181
|
-
for sigma, sigma_low in zip(sigmas, sigma_lows)
|
|
184
|
+
for sigma, sigma_low in zip(sigmas, sigma_lows, strict=True)
|
|
182
185
|
]
|
|
183
186
|
if len(sigmas) == 1:
|
|
184
187
|
return dens[0]
|
|
@@ -233,7 +236,7 @@ def nearest_higher_density_neighbor(
|
|
|
233
236
|
def remove_border_points(
|
|
234
237
|
labels, density, kdtree, search_radius=1.0, n_neighbors_search=5, workers=1
|
|
235
238
|
):
|
|
236
|
-
|
|
239
|
+
_distances, indices = kdtree.query(
|
|
237
240
|
kdtree.data,
|
|
238
241
|
k=1 + n_neighbors_search,
|
|
239
242
|
distance_upper_bound=search_radius,
|
|
@@ -249,7 +252,7 @@ def remove_border_points(
|
|
|
249
252
|
|
|
250
253
|
units = np.unique(labels)
|
|
251
254
|
new_labels = labels.copy()
|
|
252
|
-
for
|
|
255
|
+
for u in units[units > 0]:
|
|
253
256
|
in_unit = np.flatnonzero(labels == u)
|
|
254
257
|
in_unit_border = in_border[labels_in_border == u]
|
|
255
258
|
if not in_unit_border.size:
|
|
@@ -266,7 +269,7 @@ def sort_density(
|
|
|
266
269
|
sigma0: float,
|
|
267
270
|
sigma1: float,
|
|
268
271
|
max_sigma: float = 3.0,
|
|
269
|
-
device: torch.device =
|
|
272
|
+
device: torch.device | str = "cpu",
|
|
270
273
|
batch_size: int = 2048,
|
|
271
274
|
col_batch_size: int = 1024,
|
|
272
275
|
show_progress: bool = False,
|
|
@@ -295,6 +298,7 @@ def sort_density(
|
|
|
295
298
|
np.ndarray
|
|
296
299
|
The density ratio for each point in X.
|
|
297
300
|
"""
|
|
301
|
+
device = torch.device(device)
|
|
298
302
|
X = torch.asarray(X, device=device)
|
|
299
303
|
order = torch.argsort(X[:, 0])
|
|
300
304
|
X = X[order]
|
|
@@ -313,7 +317,7 @@ def sort_density(
|
|
|
313
317
|
n1 = X.new_tensor(-0.5 * dim * np.log(2.0 * np.pi * sigma1 * sigma1))
|
|
314
318
|
|
|
315
319
|
if show_progress:
|
|
316
|
-
iter = progrange(0, n, batch_size, desc=f
|
|
320
|
+
iter = progrange(0, n, batch_size, desc=f"SortDens:{device.type}")
|
|
317
321
|
else:
|
|
318
322
|
iter = range(0, n, batch_size)
|
|
319
323
|
|
|
@@ -384,7 +388,7 @@ def kdt_density(
|
|
|
384
388
|
max_sigma: float = 3.0,
|
|
385
389
|
n_threads=8,
|
|
386
390
|
):
|
|
387
|
-
n
|
|
391
|
+
n = X.shape[0]
|
|
388
392
|
max_dist = max_sigma * sigma_regional
|
|
389
393
|
|
|
390
394
|
jobs = range(0, n, batch_size)
|
|
@@ -506,7 +510,7 @@ def density_peaks(
|
|
|
506
510
|
leafsize=24,
|
|
507
511
|
density_strategy="sort",
|
|
508
512
|
workers=-1,
|
|
509
|
-
device: torch.device =
|
|
513
|
+
device: torch.device | str = "cpu",
|
|
510
514
|
):
|
|
511
515
|
"""Density peaks clustering as described by Rodriguez and Laio, but...
|
|
512
516
|
|
|
@@ -584,7 +588,7 @@ def density_peaks(
|
|
|
584
588
|
has_nhdn = np.flatnonzero(nhdn < n).astype(np.intc)
|
|
585
589
|
data = np.ones(has_nhdn.size)
|
|
586
590
|
graph = coo_array((data, (nhdn[has_nhdn], has_nhdn)), shape=(n, n))
|
|
587
|
-
|
|
591
|
+
_ncc, labels = connected_components(graph)
|
|
588
592
|
|
|
589
593
|
if remove_borders:
|
|
590
594
|
labels = remove_border_points(
|
|
@@ -724,7 +728,7 @@ def gmm_density_peaks(
|
|
|
724
728
|
|
|
725
729
|
Xi = X[choices]
|
|
726
730
|
nc = len(Xi)
|
|
727
|
-
inliers,
|
|
731
|
+
inliers, _kdtree = kdtree_inliers(
|
|
728
732
|
Xi,
|
|
729
733
|
n_neighbors=outlier_neighbor_count,
|
|
730
734
|
distance_upper_bound=outlier_radius * np.sqrt(X.shape[1]),
|
|
@@ -735,7 +739,7 @@ def gmm_density_peaks(
|
|
|
735
739
|
|
|
736
740
|
Xi = X[inliers]
|
|
737
741
|
ni = len(Xi)
|
|
738
|
-
|
|
742
|
+
_uchans, cchans = np.unique(channels[inliers], return_counts=True)
|
|
739
743
|
comps_per_chan = np.minimum(
|
|
740
744
|
max_components_per_channel,
|
|
741
745
|
np.ceil(cchans / min_spikes_per_component).astype(int),
|
|
@@ -1211,7 +1215,7 @@ def _density_peaks_clustering_uhd_implementation(
|
|
|
1211
1215
|
)
|
|
1212
1216
|
|
|
1213
1217
|
graph = coo_array((np.ones(2 * has_nhdn.size), rc), shape=(n, n))
|
|
1214
|
-
|
|
1218
|
+
_ncc, labels = connected_components(graph, directed=False)
|
|
1215
1219
|
|
|
1216
1220
|
if remove_borders:
|
|
1217
1221
|
labels = remove_border_points(
|
|
@@ -3,7 +3,7 @@ import torch
|
|
|
3
3
|
import torch.nn.functional as F
|
|
4
4
|
from torch import Tensor
|
|
5
5
|
|
|
6
|
-
from
|
|
6
|
+
from ..util.py_util import databag, panic
|
|
7
7
|
|
|
8
8
|
try:
|
|
9
9
|
import cupy # type: ignore # ty: ignore[x]
|
|
@@ -58,7 +58,7 @@ def kmeanspp(
|
|
|
58
58
|
closest = torch.cdist(X, X.mean(0, keepdim=True)).argmax()
|
|
59
59
|
centroid_ixs[0] = closest.item()
|
|
60
60
|
else:
|
|
61
|
-
|
|
61
|
+
panic()
|
|
62
62
|
|
|
63
63
|
if Xnormsq is None:
|
|
64
64
|
Xnormsq = torch.linalg.vector_norm(X, dim=1).square_()
|
|
@@ -227,7 +227,7 @@ def truncated_kmeans(
|
|
|
227
227
|
log_likelihoods = None
|
|
228
228
|
|
|
229
229
|
if show_progress:
|
|
230
|
-
it = progrange(n_iter, desc=f"kmeans
|
|
230
|
+
it = progrange(n_iter, desc=f"kmeans sigma={sigma:0.4f}")
|
|
231
231
|
else:
|
|
232
232
|
it = range(n_iter)
|
|
233
233
|
|
|
@@ -323,7 +323,7 @@ def truncated_kmeans(
|
|
|
323
323
|
break
|
|
324
324
|
prev_sigma = sigma
|
|
325
325
|
if show_progress:
|
|
326
|
-
it.set_description(f"kmeans
|
|
326
|
+
it.set_description(f"kmeans sigma={sigma:0.4f}") # type: ignore
|
|
327
327
|
|
|
328
328
|
return dict(
|
|
329
329
|
centroid_ixs=centroid_ixs,
|
|
@@ -447,7 +447,7 @@ def kmeans(
|
|
|
447
447
|
if weights is not None:
|
|
448
448
|
weights = weights.to(device=X.device)
|
|
449
449
|
with torch.jit.optimized_execution(True):
|
|
450
|
-
for
|
|
450
|
+
for _ in range(n_kmeans_tries):
|
|
451
451
|
aa, ee, cc, dists = kmeans_inner(
|
|
452
452
|
X,
|
|
453
453
|
n_kmeanspp_tries=n_kmeanspp_tries,
|
|
@@ -810,7 +810,7 @@ def truncated_kmeans_from_labels(
|
|
|
810
810
|
distsq_buf = (distsq_buf, torch.zeros_like(distsq_buf))
|
|
811
811
|
|
|
812
812
|
if show_progress:
|
|
813
|
-
it = progrange(n_iter, desc=f"kmeans
|
|
813
|
+
it = progrange(n_iter, desc=f"kmeans sigma={sigma:0.4f}")
|
|
814
814
|
else:
|
|
815
815
|
it = range(n_iter)
|
|
816
816
|
|
|
@@ -891,7 +891,7 @@ def truncated_kmeans_from_labels(
|
|
|
891
891
|
|
|
892
892
|
prev_sigma = sigma
|
|
893
893
|
if show_progress:
|
|
894
|
-
it.set_description(f"kmeans
|
|
894
|
+
it.set_description(f"kmeans sigma={sigma:0.4f}") # type: ignore
|
|
895
895
|
|
|
896
896
|
return KMeansResult(
|
|
897
897
|
labels=labels,
|
|
@@ -185,7 +185,7 @@ def calculate_merge_distances(
|
|
|
185
185
|
np.fill_diagonal(dists, 0.0) # sometimes numerical 0 is -1e-6.
|
|
186
186
|
min_dist = dists.min()
|
|
187
187
|
if min_dist < -1e-3:
|
|
188
|
-
warnings.warn(f"Alarmingly negative min distance {min_dist}.")
|
|
188
|
+
warnings.warn(f"Alarmingly negative min distance {min_dist}.", stacklevel=2)
|
|
189
189
|
dists = np.maximum(dists, 0.0, out=dists)
|
|
190
190
|
|
|
191
191
|
return units, dists, shifts, template_snrs
|