DARTsort 0.5.30__tar.gz → 0.5.31__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.30 → dartsort-0.5.31}/.github/workflows/ci.yml +1 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/.github/workflows/ci_windows.yml +1 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/PKG-INFO +1 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/clustering/agglomerate.py +338 -153
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/clustering/cluster_util.py +3 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/clustering/mixture.py +123 -22
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/clustering/refine_util.py +4 -9
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/config.py +10 -7
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/detect/detect.py +6 -7
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/evaluate/comparison.py +1 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/evaluate/noise_recording_tools.py +1 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/localize/localize_util.py +1 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/main.py +2 -2
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/matching_util/compressed_upsampled.py +19 -19
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/peel_base.py +101 -76
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/shaver.py +5 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/subtract.py +3 -9
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/threshold.py +5 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/pipeline.py +39 -30
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/data_util.py +87 -92
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/internal_config.py +21 -31
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/multiprocessing_util.py +34 -2
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/preprocess_util.py +1 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/spiketorch.py +29 -21
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/torch_util.py +5 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/waveform_util.py +1 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort.egg-info/SOURCES.txt +1 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_agglomerate.py +103 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_config.py +6 -2
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_data_util.py +45 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_hybridkit.py +1 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_modeling.py +21 -7
- dartsort-0.5.31/tests/test_multiprocessing_util.py +30 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_threshold.py +28 -1
- {dartsort-0.5.30 → dartsort-0.5.31}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/.gitignore +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/README.md +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/docs/index.md +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/docs/main_api.md +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/environment.yml +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/notebook/.keep +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/notebook/fit_and_batch_matching.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/notebook/ibl_dandi_example.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/pyproject.toml +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/setup.cfg +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/__main__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/clustering/clustering.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/clustering/clustering_features.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/clustering/kmeans.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/evaluate/analysis.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/evaluate/hybridkit.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/evaluate/simkit.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/matching.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/peel_lib.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/peel/subtract_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/decollider.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/matched_filter_net.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/mixture_classifier.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/transform_base.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/vq_matched_filter.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/main_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/motion.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/noise_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/analysis_plots.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/mixture.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/vismain.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/conftest.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_alignment.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_clustering.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_clustering_features.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_density.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_detect.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_linalg.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_matching.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_simkit.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_subtract.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_templates.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_transform.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.30 → dartsort-0.5.31}/zensical.toml +0 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Agglomeration of clusters to fix up GMM oversplits."""
|
|
2
2
|
|
|
3
|
+
import importlib.util
|
|
3
4
|
from threading import local
|
|
4
5
|
from typing import cast
|
|
5
6
|
|
|
@@ -8,13 +9,6 @@ import numpy as np
|
|
|
8
9
|
import torch
|
|
9
10
|
from spikeinterface.core import BaseRecording
|
|
10
11
|
|
|
11
|
-
try:
|
|
12
|
-
import KDEpy
|
|
13
|
-
|
|
14
|
-
HAVE_KDEPY = True
|
|
15
|
-
except ImportError:
|
|
16
|
-
HAVE_KDEPY = False
|
|
17
|
-
|
|
18
12
|
from ..templates.template_util import shared_basis_compress_templates
|
|
19
13
|
from ..templates.templates import TemplateData
|
|
20
14
|
from ..util.data_util import (
|
|
@@ -55,6 +49,7 @@ from .cluster_util import (
|
|
|
55
49
|
)
|
|
56
50
|
|
|
57
51
|
logger = get_logger(__name__)
|
|
52
|
+
HAVE_KDEPY = importlib.util.find_spec("KDEpy") is not None
|
|
58
53
|
|
|
59
54
|
|
|
60
55
|
@databag
|
|
@@ -64,7 +59,6 @@ class Agglomeration:
|
|
|
64
59
|
template_distances: np.ndarray | None
|
|
65
60
|
template_shifts: np.ndarray | None
|
|
66
61
|
violation: ViolationInfo | None
|
|
67
|
-
glom_cost: np.ndarray | None
|
|
68
62
|
|
|
69
63
|
|
|
70
64
|
def agglomerate(
|
|
@@ -87,17 +81,25 @@ def agglomerate(
|
|
|
87
81
|
If refinement_cfg is not set, the merge is just a hierarchical clustering
|
|
88
82
|
of the template distances.
|
|
89
83
|
|
|
90
|
-
The algorithm is like this.
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
84
|
+
The algorithm is like this (violation_linkage). It's a greedy procedure building
|
|
85
|
+
up a custom linkage as in hierarchical clustering based on chance-corrected refractory
|
|
86
|
+
violations and template distances both.
|
|
87
|
+
- For groups A and B, let d be the distance between their merged templates.
|
|
88
|
+
Let E, O be their expected and observed violation counts summed over member
|
|
89
|
+
pairs with E under the jitter model of violation_statistics.
|
|
90
|
+
- A pair of groups is a candidate for merging if either of:
|
|
91
|
+
- d < glom_force_merge_template_distance
|
|
92
|
+
- d < merge_distance_threshold and E >= glom_min_violation_evidence.
|
|
93
|
+
- Additionally, each member unit u of A must have, against the whole of B,
|
|
94
|
+
at least one of:
|
|
95
|
+
- E(u, B) >= glom_min_violation_evidence
|
|
96
|
+
- Distance from u's template to B's < glom_force_merge_template_distance
|
|
97
|
+
- Groups are merged greedily in chance-corrected violation ratio (O/E) order
|
|
98
|
+
up to glom_violation_threshold
|
|
99
|
+
- Optionally, groups are then split so that no pair of units within a
|
|
100
|
+
group has violation ratio >= glom_veto_threshold.
|
|
101
|
+
- If glom_violation_threshold is None, groups are the connected components
|
|
102
|
+
of the allowed-pair graph by template distance according to the linkage parameter.
|
|
101
103
|
|
|
102
104
|
This code might read a little weird, because it's used both as a clustering pass
|
|
103
105
|
and to apply a distance-based merge to the template library. In the latter case
|
|
@@ -122,7 +124,6 @@ def agglomerate(
|
|
|
122
124
|
template_distances=None,
|
|
123
125
|
template_shifts=None,
|
|
124
126
|
violation=None,
|
|
125
|
-
glom_cost=None,
|
|
126
127
|
)
|
|
127
128
|
|
|
128
129
|
tdist = template_distances(
|
|
@@ -137,43 +138,31 @@ def agglomerate(
|
|
|
137
138
|
distances = np.minimum(tdist.distances, tdist.distances.T)
|
|
138
139
|
np.fill_diagonal(distances, 0.0)
|
|
139
140
|
assert np.array_equal(tdist.template_data.unit_ids, np.arange(distances.shape[0]))
|
|
141
|
+
unit_snrs = tdist.template_data.snrs_by_channel().max(1)
|
|
140
142
|
|
|
141
|
-
# early out
|
|
142
143
|
if refinement_cfg is None:
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
144
|
+
_, merge_mapping = hierarchical_cluster(
|
|
145
|
+
None,
|
|
146
|
+
distances,
|
|
147
|
+
linkage_method=template_merge_cfg.linkage,
|
|
148
|
+
threshold=template_merge_cfg.merge_distance_threshold,
|
|
149
|
+
)
|
|
147
150
|
violation = None
|
|
148
151
|
else:
|
|
149
|
-
|
|
150
|
-
sorting,
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
linkage_method,
|
|
157
|
-
threshold,
|
|
158
|
-
veto_threshold,
|
|
159
|
-
violation,
|
|
160
|
-
) = res
|
|
161
|
-
|
|
162
|
-
_, merge_mapping = hierarchical_cluster(
|
|
163
|
-
None, glom_cost, linkage_method=linkage_method, threshold=threshold
|
|
164
|
-
)
|
|
165
|
-
if veto_cost is not None:
|
|
166
|
-
assert veto_threshold is not None
|
|
167
|
-
_, veto_mapping = hierarchical_cluster(
|
|
168
|
-
None, veto_cost, linkage_method="complete", threshold=veto_threshold
|
|
152
|
+
merge_mapping, violation = _agglomerate_violation_merge(
|
|
153
|
+
sorting=sorting,
|
|
154
|
+
distances=distances,
|
|
155
|
+
tdist=tdist,
|
|
156
|
+
unit_snrs=unit_snrs,
|
|
157
|
+
refinement_cfg=refinement_cfg,
|
|
158
|
+
template_merge_cfg=template_merge_cfg,
|
|
169
159
|
)
|
|
170
|
-
merge_mapping = meet(merge_mapping, veto_mapping)
|
|
171
160
|
|
|
172
161
|
agg_sorting = apply_reclustering(
|
|
173
162
|
sorting=sorting,
|
|
174
163
|
merge_mapping=merge_mapping,
|
|
175
164
|
shifts=tdist.shifts,
|
|
176
|
-
unit_snrs=
|
|
165
|
+
unit_snrs=unit_snrs,
|
|
177
166
|
in_place=in_place,
|
|
178
167
|
)
|
|
179
168
|
agg_sorting = combine_gmm_scores(
|
|
@@ -189,75 +178,30 @@ def agglomerate(
|
|
|
189
178
|
template_distances=distances,
|
|
190
179
|
template_shifts=tdist.shifts,
|
|
191
180
|
violation=violation,
|
|
192
|
-
glom_cost=glom_cost,
|
|
193
181
|
)
|
|
194
182
|
|
|
195
183
|
|
|
196
184
|
def _agglomerate_violation_merge(
|
|
185
|
+
*,
|
|
197
186
|
sorting: DARTsortSorting,
|
|
198
187
|
distances: np.ndarray,
|
|
188
|
+
tdist: "TemplateDistanceResult",
|
|
189
|
+
unit_snrs: np.ndarray,
|
|
199
190
|
refinement_cfg: RefinementConfig,
|
|
200
191
|
template_merge_cfg: TemplateMergeConfig,
|
|
201
|
-
|
|
202
|
-
huge=1e8,
|
|
203
|
-
) -> tuple[
|
|
204
|
-
np.ndarray,
|
|
205
|
-
np.ndarray,
|
|
206
|
-
np.ndarray | None,
|
|
207
|
-
str,
|
|
208
|
-
float,
|
|
209
|
-
float | None,
|
|
210
|
-
ViolationInfo | None,
|
|
211
|
-
]:
|
|
212
|
-
mask = distances < template_merge_cfg.merge_distance_threshold
|
|
213
|
-
|
|
214
|
-
if refinement_cfg.glom_qda_overlap or refinement_cfg.glom_qda_bimodality:
|
|
215
|
-
qda_res = qda(
|
|
216
|
-
mask=mask,
|
|
217
|
-
sorting=sorting,
|
|
218
|
-
min_iou=refinement_cfg.qda_min_iou
|
|
219
|
-
if refinement_cfg.glom_qda_overlap
|
|
220
|
-
else 0.0,
|
|
221
|
-
min_cov=refinement_cfg.qda_min_coverage
|
|
222
|
-
if refinement_cfg.glom_qda_overlap
|
|
223
|
-
else 0.0,
|
|
224
|
-
bimodality=refinement_cfg.glom_qda_bimodality,
|
|
225
|
-
show_progress=False,
|
|
226
|
-
computation_cfg=computation_cfg,
|
|
227
|
-
)
|
|
228
|
-
if refinement_cfg.glom_qda_overlap:
|
|
229
|
-
mask &= np.logical_and(
|
|
230
|
-
qda_res.coverage >= refinement_cfg.qda_min_coverage,
|
|
231
|
-
qda_res.iou >= refinement_cfg.qda_min_iou,
|
|
232
|
-
)
|
|
233
|
-
if refinement_cfg.glom_qda_bimodality:
|
|
234
|
-
mask &= np.logical_or(
|
|
235
|
-
qda_res.score >= refinement_cfg.qda_uni_score,
|
|
236
|
-
np.logical_and(
|
|
237
|
-
qda_res.score >= refinement_cfg.qda_threshold,
|
|
238
|
-
qda_res.min_ratio >= refinement_cfg.qda_min_ratio,
|
|
239
|
-
),
|
|
240
|
-
)
|
|
241
|
-
|
|
242
|
-
force_mask = distances < refinement_cfg.glom_force_merge_template_distance
|
|
243
|
-
mask |= force_mask
|
|
244
|
-
np.fill_diagonal(mask, True)
|
|
245
|
-
|
|
192
|
+
) -> tuple[np.ndarray, ViolationInfo | None]:
|
|
246
193
|
# early out: no violation stuff. just distance mask connected components.
|
|
247
194
|
if refinement_cfg.glom_violation_threshold is None:
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
return (
|
|
253
|
-
mask,
|
|
254
|
-
glom_cost,
|
|
255
|
-
veto_cost,
|
|
256
|
-
linkage_method,
|
|
257
|
-
threshold,
|
|
258
|
-
veto_threshold,
|
|
195
|
+
mask = distances < template_merge_cfg.merge_distance_threshold
|
|
196
|
+
mask |= distances < refinement_cfg.glom_force_merge_template_distance
|
|
197
|
+
np.fill_diagonal(mask, True)
|
|
198
|
+
_, merge_mapping = hierarchical_cluster(
|
|
259
199
|
None,
|
|
200
|
+
np.logical_not(mask).astype(np.float32),
|
|
201
|
+
linkage_method=template_merge_cfg.linkage,
|
|
202
|
+
threshold=0.5,
|
|
260
203
|
)
|
|
204
|
+
return merge_mapping, None
|
|
261
205
|
|
|
262
206
|
violation = violation_statistics(
|
|
263
207
|
sorting,
|
|
@@ -265,52 +209,252 @@ def _agglomerate_violation_merge(
|
|
|
265
209
|
viol_ms=refinement_cfg.glom_violation_ms,
|
|
266
210
|
jitter_ms=refinement_cfg.glom_jitter_ms,
|
|
267
211
|
)
|
|
268
|
-
assert
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
212
|
+
assert sorting.labels is not None
|
|
213
|
+
spike_counts = np.bincount(
|
|
214
|
+
sorting.labels[sorting.labels >= 0], minlength=distances.shape[0]
|
|
215
|
+
)
|
|
216
|
+
merge_mapping = violation_linkage(
|
|
217
|
+
distances=distances,
|
|
218
|
+
violation=violation,
|
|
219
|
+
tdist=tdist,
|
|
220
|
+
spike_counts=spike_counts,
|
|
221
|
+
unit_snrs=unit_snrs,
|
|
222
|
+
template_merge_cfg=template_merge_cfg,
|
|
223
|
+
force_distance=refinement_cfg.glom_force_merge_template_distance,
|
|
224
|
+
min_evidence=refinement_cfg.glom_min_violation_evidence,
|
|
225
|
+
violation_threshold=refinement_cfg.glom_violation_threshold,
|
|
277
226
|
)
|
|
278
|
-
# mask out distance enemies
|
|
279
|
-
glom_cost[np.logical_not(mask)] = huge
|
|
280
|
-
glom_cost = np.minimum(glom_cost, glom_cost.T)
|
|
281
|
-
np.fill_diagonal(glom_cost, 0.0)
|
|
282
227
|
|
|
283
228
|
# last thing: optionally, be extremely finnicky about merging into violated groups
|
|
284
229
|
# i am not sure if this will be a good idea or not; it may cost too many merges
|
|
285
230
|
# to be worthwhile.
|
|
286
231
|
if refinement_cfg.glom_veto_threshold is not None:
|
|
287
232
|
veto_cost = violation.jitter_viol_ratio(
|
|
288
|
-
refinement_cfg.
|
|
233
|
+
refinement_cfg.glom_min_violation_evidence, fill_value=0.0
|
|
289
234
|
)
|
|
290
|
-
veto_cost[np.logical_not(mask)] = huge
|
|
291
235
|
veto_cost = np.minimum(veto_cost, veto_cost.T)
|
|
292
236
|
np.fill_diagonal(veto_cost, 0.0)
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
237
|
+
_, veto_mapping = hierarchical_cluster(
|
|
238
|
+
None,
|
|
239
|
+
veto_cost,
|
|
240
|
+
linkage_method="complete",
|
|
241
|
+
threshold=refinement_cfg.glom_veto_threshold,
|
|
242
|
+
)
|
|
243
|
+
merge_mapping = meet(merge_mapping, veto_mapping)
|
|
244
|
+
|
|
245
|
+
return merge_mapping, violation
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def violation_linkage(
|
|
249
|
+
*,
|
|
250
|
+
distances: np.ndarray,
|
|
251
|
+
violation: ViolationInfo,
|
|
252
|
+
tdist: "TemplateDistanceResult",
|
|
253
|
+
spike_counts: np.ndarray,
|
|
254
|
+
unit_snrs: np.ndarray,
|
|
255
|
+
template_merge_cfg: TemplateMergeConfig,
|
|
256
|
+
force_distance: float,
|
|
257
|
+
min_evidence: float,
|
|
258
|
+
violation_threshold: float,
|
|
259
|
+
) -> np.ndarray:
|
|
260
|
+
"""Greedy agglomeration on chance-corrected violations and merged templates
|
|
261
|
+
|
|
262
|
+
Groups' merged templates are the spike-count-weighted average of their
|
|
263
|
+
members' templates, aligned to the group's highest-SNR member as in
|
|
264
|
+
apply_reclustering.
|
|
265
|
+
- For groups A and B, let d be the distance between their merged templates.
|
|
266
|
+
Let E, O be their expected and observed violation counts summed over member
|
|
267
|
+
pairs.
|
|
268
|
+
- A pair of groups is a candidate for merging if either of:
|
|
269
|
+
- d < force_distance
|
|
270
|
+
- d < merge_distance_threshold and E >= min_evidence.
|
|
271
|
+
- Additionally, each member unit u of A must have, against the whole of B,
|
|
272
|
+
at least one of:
|
|
273
|
+
- E(u, B) >= min_evidence
|
|
274
|
+
- Distance from u's template to B's < force_distance
|
|
275
|
+
- Groups are merged greedily in O/E order (0 if E < min_evidence) up to
|
|
276
|
+
violation_threshold
|
|
277
|
+
|
|
278
|
+
Returns the merge mapping (group label of each unit).
|
|
279
|
+
"""
|
|
280
|
+
n = distances.shape[0]
|
|
281
|
+
assert violation.jitter_counts is not None
|
|
282
|
+
assert distances.shape == violation.viol_counts.shape == (n, n)
|
|
283
|
+
assert np.array_equal(distances, distances.T)
|
|
284
|
+
assert not np.isnan(distances).any()
|
|
285
|
+
assert (np.diagonal(distances) == 0).all()
|
|
286
|
+
assert spike_counts.shape == unit_snrs.shape == (n,)
|
|
287
|
+
assert (spike_counts > 0).all()
|
|
288
|
+
assert np.isfinite(unit_snrs).all()
|
|
289
|
+
assert tdist.radial_counts is not None
|
|
290
|
+
|
|
291
|
+
templates = tdist.template_data.templates
|
|
292
|
+
assert templates.shape[0] == n
|
|
293
|
+
assert tdist.temporal_components.shape[1] == templates.shape[1]
|
|
294
|
+
assert tdist.radial_counts.shape == (n, templates.shape[2])
|
|
295
|
+
assert tdist.shifts.shape == (n, n)
|
|
296
|
+
unit_spatial_sing = np.einsum("rt,ntc->nrc", tdist.temporal_components, templates)
|
|
297
|
+
unit_weights = radial_weights(tdist.radial_counts)
|
|
298
|
+
|
|
299
|
+
# groups are identified by their lowest unit id. every unit starts as its own
|
|
300
|
+
# group, so group stats start as unit stats.
|
|
301
|
+
members = {u: [u] for u in range(n)}
|
|
302
|
+
group_spatial_sing = unit_spatial_sing.copy()
|
|
303
|
+
group_weights = unit_weights.copy()
|
|
304
|
+
group_dist = distances.copy()
|
|
305
|
+
group_obs_viol = violation.viol_counts.astype(np.float64)
|
|
306
|
+
group_exp_viol = violation.jitter_counts.astype(np.float64)
|
|
307
|
+
# rows: units, cols: current groups (members.keys()). a unit's entry for its
|
|
308
|
+
# own group is stale; it's never read.
|
|
309
|
+
unit_to_group_dist = distances.copy()
|
|
310
|
+
unit_to_group_exp_viol = group_exp_viol.copy()
|
|
311
|
+
|
|
312
|
+
while True:
|
|
313
|
+
group_ids = np.array(sorted(members))
|
|
314
|
+
dist = group_dist[group_ids][:, group_ids]
|
|
315
|
+
exp_viol = group_exp_viol[group_ids][:, group_ids]
|
|
316
|
+
obs_viol = group_obs_viol[group_ids][:, group_ids]
|
|
317
|
+
|
|
318
|
+
# O/E for candidate pairs (0 if E < min_evidence), inf for the rest
|
|
319
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
320
|
+
ratio = obs_viol / exp_viol
|
|
321
|
+
ratio[exp_viol < min_evidence] = 0.0
|
|
322
|
+
allow = dist < force_distance
|
|
323
|
+
allow |= np.logical_and(
|
|
324
|
+
dist < template_merge_cfg.merge_distance_threshold,
|
|
325
|
+
exp_viol >= min_evidence,
|
|
326
|
+
)
|
|
327
|
+
ratio[np.logical_not(allow, out=allow)] = np.inf
|
|
328
|
+
del allow
|
|
329
|
+
|
|
330
|
+
# find lowest-ratio group pair below violation_threshold which passes the
|
|
331
|
+
# per-member evidence check (or dist forcing)
|
|
332
|
+
ii, jj = np.nonzero(np.triu(ratio < violation_threshold, k=1))
|
|
333
|
+
ratio_pair_order = np.argsort(ratio[ii, jj], kind="stable")
|
|
334
|
+
for i, j in zip(ii[ratio_pair_order], jj[ratio_pair_order], strict=True):
|
|
335
|
+
a = group_ids[i]
|
|
336
|
+
b = group_ids[j]
|
|
337
|
+
assert a < b
|
|
338
|
+
if _all_group_members_have_evidence(
|
|
339
|
+
a,
|
|
340
|
+
b,
|
|
341
|
+
members=members,
|
|
342
|
+
unit_to_group_exp_viol=unit_to_group_exp_viol,
|
|
343
|
+
unit_to_group_dist=unit_to_group_dist,
|
|
344
|
+
min_evidence=min_evidence,
|
|
345
|
+
force_distance=force_distance,
|
|
346
|
+
):
|
|
347
|
+
break
|
|
348
|
+
else: # no pair passed: done
|
|
349
|
+
break
|
|
350
|
+
|
|
351
|
+
# merge B into A
|
|
352
|
+
# combine violation stats over the groups
|
|
353
|
+
members[a] += members.pop(b)
|
|
354
|
+
group_obs_viol[a] += group_obs_viol[b]
|
|
355
|
+
group_obs_viol[:, a] += group_obs_viol[:, b]
|
|
356
|
+
group_exp_viol[a] += group_exp_viol[b]
|
|
357
|
+
group_exp_viol[:, a] += group_exp_viol[:, b]
|
|
358
|
+
unit_to_group_exp_viol[:, a] += unit_to_group_exp_viol[:, b]
|
|
359
|
+
|
|
360
|
+
# recompute A's template and distances to other groups and units
|
|
361
|
+
group_spatial_sing[a], group_weights[a] = _merged_template(
|
|
362
|
+
members[a],
|
|
363
|
+
templates=templates,
|
|
364
|
+
spike_counts=spike_counts,
|
|
365
|
+
unit_snrs=unit_snrs,
|
|
366
|
+
shifts=tdist.shifts,
|
|
367
|
+
temporal_components=tdist.temporal_components,
|
|
368
|
+
radial_counts=tdist.radial_counts,
|
|
369
|
+
)
|
|
370
|
+
other_groups = [g for g in members if g != a]
|
|
371
|
+
a_dist = template_distances_to(
|
|
372
|
+
group_spatial_sing[a],
|
|
373
|
+
group_weights[a],
|
|
374
|
+
group_spatial_sing[other_groups],
|
|
375
|
+
group_weights[other_groups],
|
|
376
|
+
trimmed_tconv=tdist.trimmed_tconv,
|
|
377
|
+
template_merge_cfg=template_merge_cfg,
|
|
378
|
+
)
|
|
379
|
+
group_dist[a, other_groups] = a_dist
|
|
380
|
+
group_dist[other_groups, a] = a_dist
|
|
381
|
+
nonmembers = np.setdiff1d(np.arange(n), members[a])
|
|
382
|
+
unit_to_group_dist[nonmembers, a] = template_distances_to(
|
|
383
|
+
group_spatial_sing[a],
|
|
384
|
+
group_weights[a],
|
|
385
|
+
unit_spatial_sing[nonmembers],
|
|
386
|
+
unit_weights[nonmembers],
|
|
387
|
+
trimmed_tconv=tdist.trimmed_tconv,
|
|
388
|
+
template_merge_cfg=template_merge_cfg,
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
merge_mapping = np.full(n, -1, dtype=np.int64)
|
|
392
|
+
for label, unit_ids in enumerate(members.values()):
|
|
393
|
+
merge_mapping[unit_ids] = label
|
|
394
|
+
assert np.min(merge_mapping, initial=0) >= 0
|
|
395
|
+
return merge_mapping
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _all_group_members_have_evidence(
|
|
399
|
+
a: int,
|
|
400
|
+
b: int,
|
|
401
|
+
*,
|
|
402
|
+
members: dict[int, list[int]],
|
|
403
|
+
unit_to_group_exp_viol: np.ndarray,
|
|
404
|
+
unit_to_group_dist: np.ndarray,
|
|
405
|
+
min_evidence: float,
|
|
406
|
+
force_distance: float,
|
|
407
|
+
) -> bool:
|
|
408
|
+
"""Each unit u in A has E(u, B) >= min_evidence or d(u, B) < force_distance; and vice versa"""
|
|
409
|
+
return all(
|
|
410
|
+
np.logical_or(
|
|
411
|
+
unit_to_group_exp_viol[members[src], dst] >= min_evidence,
|
|
412
|
+
unit_to_group_dist[members[src], dst] < force_distance,
|
|
413
|
+
).all()
|
|
414
|
+
for src, dst in ((a, b), (b, a))
|
|
304
415
|
)
|
|
305
416
|
|
|
306
417
|
|
|
418
|
+
def _merged_template(
|
|
419
|
+
unit_ids,
|
|
420
|
+
*,
|
|
421
|
+
templates: np.ndarray,
|
|
422
|
+
spike_counts: np.ndarray,
|
|
423
|
+
unit_snrs: np.ndarray,
|
|
424
|
+
shifts: np.ndarray,
|
|
425
|
+
temporal_components: np.ndarray,
|
|
426
|
+
radial_counts: np.ndarray,
|
|
427
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
428
|
+
"""Shift, weighted sum in temporal mask, project onto known shared basis"""
|
|
429
|
+
unit_ids = np.sort(unit_ids)
|
|
430
|
+
ref = unit_ids[np.argmax(unit_snrs[unit_ids])]
|
|
431
|
+
n_samples = templates.shape[1]
|
|
432
|
+
total = np.zeros(templates.shape[1:])
|
|
433
|
+
coverage = np.zeros(n_samples)
|
|
434
|
+
for u in unit_ids:
|
|
435
|
+
s = shifts[ref, u]
|
|
436
|
+
assert abs(s) < n_samples
|
|
437
|
+
src = slice(max(0, -s), n_samples - max(0, s))
|
|
438
|
+
dst = slice(max(0, s), n_samples - max(0, -s))
|
|
439
|
+
total[dst] += spike_counts[u] * templates[u, src]
|
|
440
|
+
coverage[dst] += spike_counts[u]
|
|
441
|
+
assert (coverage > 0).all()
|
|
442
|
+
template = np.divide(total, coverage[:, None], out=total)
|
|
443
|
+
spatial = temporal_components @ template
|
|
444
|
+
weights = radial_weights(radial_counts[unit_ids].sum(0))
|
|
445
|
+
return spatial, weights
|
|
446
|
+
|
|
447
|
+
|
|
307
448
|
@databag
|
|
308
449
|
class TemplateDistanceResult:
|
|
309
450
|
distances: np.ndarray
|
|
310
451
|
shifts: np.ndarray
|
|
452
|
+
"""shifts[i, j] is like trough[i] - trough[j]"""
|
|
311
453
|
r2: np.ndarray
|
|
312
454
|
template_data: TemplateData
|
|
313
|
-
|
|
455
|
+
temporal_components: np.ndarray
|
|
456
|
+
trimmed_tconv: torch.Tensor
|
|
457
|
+
radial_counts: np.ndarray | None
|
|
314
458
|
spatial_iou: np.ndarray | None
|
|
315
459
|
|
|
316
460
|
|
|
@@ -399,12 +543,12 @@ def template_distances(
|
|
|
399
543
|
center = conv_len // 2
|
|
400
544
|
assert conv_len == 2 * center + 1
|
|
401
545
|
assert center >= max_shift
|
|
402
|
-
|
|
403
|
-
|
|
546
|
+
trimmed_tconv = tconv[:, :, center - max_shift : center + 1 + max_shift]
|
|
547
|
+
trimmed_tconv = trimmed_tconv.contiguous()
|
|
404
548
|
|
|
405
|
-
|
|
549
|
+
radial_counts = spatial_iou = None
|
|
406
550
|
if template_merge_cfg.distance_kind == "scaled_normeuc":
|
|
407
|
-
best_conv, best_lag = best_shared_pconv(
|
|
551
|
+
best_conv, best_lag = best_shared_pconv(trimmed_tconv, spatial_sing)
|
|
408
552
|
dist = scaled_normeuc_from_dots(
|
|
409
553
|
best_conv,
|
|
410
554
|
scale_var=template_merge_cfg.amplitude_scaling_variance,
|
|
@@ -413,14 +557,15 @@ def template_distances(
|
|
|
413
557
|
elif template_merge_cfg.distance_kind == "weighted_scaled_normeuc":
|
|
414
558
|
assert sorting is not None
|
|
415
559
|
assert motion is not None
|
|
416
|
-
|
|
560
|
+
radial_counts = radial_spike_counts(
|
|
417
561
|
sorting=sorting,
|
|
418
562
|
motion=motion,
|
|
419
563
|
radius=template_merge_cfg.weighted_dist_radius,
|
|
420
564
|
)
|
|
565
|
+
spatial_weights = radial_weights(radial_counts)
|
|
421
566
|
assert np.isfinite(spatial_weights).all()
|
|
422
567
|
dist, best_lag, iou = weighted_best_lagged_scaled_normeuc_dist(
|
|
423
|
-
tconv=
|
|
568
|
+
tconv=trimmed_tconv,
|
|
424
569
|
spatial_sing=spatial_sing,
|
|
425
570
|
weights=torch.asarray(spatial_weights).to(spatial_sing),
|
|
426
571
|
scale_var=template_merge_cfg.amplitude_scaling_variance,
|
|
@@ -434,14 +579,56 @@ def template_distances(
|
|
|
434
579
|
# okay then
|
|
435
580
|
return TemplateDistanceResult(
|
|
436
581
|
distances=dist.numpy(force=True),
|
|
437
|
-
shifts=best_lag.numpy(force=True),
|
|
582
|
+
shifts=best_lag.T.numpy(force=True),
|
|
438
583
|
r2=cast(np.ndarray, sbt.r2),
|
|
439
584
|
template_data=template_data,
|
|
440
|
-
|
|
585
|
+
temporal_components=sbt.temporal_components,
|
|
586
|
+
trimmed_tconv=trimmed_tconv,
|
|
587
|
+
radial_counts=radial_counts,
|
|
441
588
|
spatial_iou=spatial_iou,
|
|
442
589
|
)
|
|
443
590
|
|
|
444
591
|
|
|
592
|
+
def template_distances_to(
|
|
593
|
+
spatial_sing_a: np.ndarray,
|
|
594
|
+
weights_a: np.ndarray,
|
|
595
|
+
spatial_sing_b: np.ndarray,
|
|
596
|
+
weights_b: np.ndarray,
|
|
597
|
+
*,
|
|
598
|
+
trimmed_tconv: torch.Tensor,
|
|
599
|
+
template_merge_cfg: TemplateMergeConfig,
|
|
600
|
+
) -> np.ndarray:
|
|
601
|
+
assert template_merge_cfg.distance_kind == "weighted_scaled_normeuc"
|
|
602
|
+
n = spatial_sing_b.shape[0]
|
|
603
|
+
assert spatial_sing_b.shape[1:] == spatial_sing_a.shape
|
|
604
|
+
assert weights_a.shape == spatial_sing_a.shape[1:]
|
|
605
|
+
assert weights_b.shape == (n, *weights_a.shape)
|
|
606
|
+
assert weights_a.max() > 0 and (weights_b.max(1) > 0).all()
|
|
607
|
+
|
|
608
|
+
dist = np.full(n, np.inf)
|
|
609
|
+
iou = np.minimum(weights_a, weights_b).sum(1) / np.maximum(
|
|
610
|
+
weights_a, weights_b
|
|
611
|
+
).sum(1)
|
|
612
|
+
(ix,) = np.nonzero(iou >= template_merge_cfg.weighted_dist_min_iou)
|
|
613
|
+
if not ix.size:
|
|
614
|
+
return dist
|
|
615
|
+
|
|
616
|
+
spatial_sing = np.concatenate([spatial_sing_a[None], spatial_sing_b[ix]])
|
|
617
|
+
weights = np.concatenate([weights_a[None], weights_b[ix]])
|
|
618
|
+
jj = torch.arange(1, ix.size + 1)
|
|
619
|
+
d, _, pair_iou = weighted_best_lagged_scaled_normeuc_dist(
|
|
620
|
+
tconv=trimmed_tconv,
|
|
621
|
+
spatial_sing=torch.asarray(spatial_sing).to(trimmed_tconv),
|
|
622
|
+
weights=torch.asarray(weights).to(trimmed_tconv),
|
|
623
|
+
scale_var=template_merge_cfg.amplitude_scaling_variance,
|
|
624
|
+
scale_boundary=template_merge_cfg.amplitude_scaling_boundary,
|
|
625
|
+
pairs=(torch.zeros_like(jj), jj),
|
|
626
|
+
)
|
|
627
|
+
d.masked_fill_(pair_iou < template_merge_cfg.weighted_dist_min_iou, torch.inf)
|
|
628
|
+
dist[ix] = d.numpy(force=True)
|
|
629
|
+
return dist
|
|
630
|
+
|
|
631
|
+
|
|
445
632
|
@databag
|
|
446
633
|
class QDAResult:
|
|
447
634
|
"""Unit pair QDA metrics
|
|
@@ -689,7 +876,7 @@ def bimod_stats(h):
|
|
|
689
876
|
return a, b
|
|
690
877
|
|
|
691
878
|
|
|
692
|
-
def
|
|
879
|
+
def radial_spike_counts(sorting: DARTsortSorting, motion: MotionInfo, radius: float):
|
|
693
880
|
assert sorting.labels is not None
|
|
694
881
|
kept = np.flatnonzero(sorting.labels >= 0)
|
|
695
882
|
|
|
@@ -708,23 +895,21 @@ def count_radial_weights(sorting: DARTsortSorting, motion: MotionInfo, radius: f
|
|
|
708
895
|
ci = make_channel_index(motion.rgeom, radius, to_torch=False)
|
|
709
896
|
|
|
710
897
|
# puff out with radial neighborhood and sum up the counts by label
|
|
711
|
-
|
|
898
|
+
radial_counts = np.zeros(counts.shape)
|
|
712
899
|
for uu in range(counts.shape[0]):
|
|
713
900
|
row = counts[uu]
|
|
714
901
|
ii = np.flatnonzero(row)
|
|
715
|
-
|
|
716
|
-
continue
|
|
717
|
-
vv = row[ii]
|
|
718
|
-
vv = vv / vv.sum()
|
|
719
|
-
|
|
720
|
-
for channel, value in zip(ii, vv, strict=True):
|
|
902
|
+
for channel, value in zip(ii, row[ii], strict=True):
|
|
721
903
|
cixs = ci[channel]
|
|
722
904
|
cixs = cixs[cixs < motion.rgeom.shape[0]]
|
|
723
|
-
|
|
905
|
+
radial_counts[uu, cixs] += value
|
|
906
|
+
|
|
907
|
+
return radial_counts
|
|
908
|
+
|
|
724
909
|
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
return
|
|
910
|
+
def radial_weights(radial_counts: np.ndarray) -> np.ndarray:
|
|
911
|
+
denom = radial_counts.max(axis=-1, keepdims=True).clip(min=1e-10)
|
|
912
|
+
return radial_counts / denom
|
|
728
913
|
|
|
729
914
|
|
|
730
915
|
def combine_gmm_scores(
|
|
@@ -923,7 +1108,7 @@ def clean_final_sorting(
|
|
|
923
1108
|
motion: MotionInfo,
|
|
924
1109
|
dedup_ms: float = -1.0,
|
|
925
1110
|
merge_mapping: np.ndarray | None = None,
|
|
926
|
-
score_by=("
|
|
1111
|
+
score_by=("gmm_log_liks", "scores"),
|
|
927
1112
|
in_place: bool = True,
|
|
928
1113
|
) -> tuple[DARTsortSorting, np.ndarray]:
|
|
929
1114
|
"""Deduplicate, flatten, depth-order
|
|
@@ -961,7 +1146,7 @@ def clean_final_sorting(
|
|
|
961
1146
|
def deduplicate_spikes(
|
|
962
1147
|
sorting: DARTsortSorting,
|
|
963
1148
|
radius_ms: float = -1.0,
|
|
964
|
-
score_by=("
|
|
1149
|
+
score_by=("gmm_log_liks", "scores"),
|
|
965
1150
|
in_place: bool = False,
|
|
966
1151
|
) -> DARTsortSorting:
|
|
967
1152
|
if radius_ms < 0 or sorting.labels is None:
|