DARTsort 0.5.22__tar.gz → 0.5.23__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.22 → dartsort-0.5.23}/PKG-INFO +1 -1
- {dartsort-0.5.22 → dartsort-0.5.23}/pyproject.toml +1 -2
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/__init__.py +6 -1
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/clustering/agglomerate.py +1 -1
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/clustering/cluster_util.py +105 -2
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/clustering/density.py +2 -2
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/clustering/merge.py +32 -6
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/clustering/mixture.py +21 -12
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/config.py +1 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/evaluate/analysis.py +25 -27
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/evaluate/comparison.py +22 -11
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/evaluate/hybrid_util.py +22 -16
- dartsort-0.5.23/src/dartsort/evaluate/hybridkit.py +286 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/evaluate/sim_template_tools.py +139 -41
- dartsort-0.5.23/src/dartsort/evaluate/simkit.py +372 -0
- dartsort-0.5.23/src/dartsort/evaluate/simlib.py +1063 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/main.py +58 -66
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/matching.py +3 -2
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/peel_base.py +5 -12
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/peel_lib.py +3 -4
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/reduction_template.py +5 -9
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/subtract.py +1 -1
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/templates/template_util.py +2 -2
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/templates/templates.py +14 -14
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/mixture_classifier.py +2 -2
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/reduction.py +1 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/data_util.py +58 -40
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/internal_config.py +27 -18
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/logging_util.py +8 -8
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/main_util.py +49 -16
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/motion.py +41 -4
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/multiprocessing_util.py +1 -1
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/peel_util.py +0 -2
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/py_util.py +12 -7
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/torch_util.py +1 -4
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/gt.py +90 -78
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/scatterplots.py +7 -5
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/sorting.py +2 -1
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/unit_comparison.py +21 -18
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort.egg-info/SOURCES.txt +2 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_alignment.py +3 -0
- dartsort-0.5.23/tests/test_cluster_util.py +209 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_data_util.py +1 -1
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_grab_and_featurize.py +3 -4
- dartsort-0.5.23/tests/test_hybridkit.py +432 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_simkit.py +6 -5
- dartsort-0.5.22/src/dartsort/evaluate/simkit.py +0 -911
- dartsort-0.5.22/src/dartsort/evaluate/simlib.py +0 -362
- dartsort-0.5.22/tests/test_cluster_util.py +0 -83
- {dartsort-0.5.22 → dartsort-0.5.23}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/.gitignore +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/README.md +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/docs/index.md +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/docs/main_api.md +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/environment.yml +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/notebook/.keep +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/notebook/ibl_dandi_example.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/setup.cfg +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/__main__.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/clustering/clustering.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/clustering/clustering_features.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/clustering/kmeans.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/clustering/refine_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/matching_util/__init__.py +6 -6
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/decollider.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/pipeline.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/transform_base.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/noise_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/spiketorch.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/analysis_plots.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/mixture.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/vismain.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/conftest.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_agglomerate.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_clustering.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_config.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_density.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_detect.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_linalg.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_matching.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_modeling.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_subtract.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_templates.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_threshold.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_transform.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.22 → dartsort-0.5.23}/zensical.toml +0 -0
|
@@ -75,7 +75,7 @@ line-length = 88
|
|
|
75
75
|
|
|
76
76
|
[tool.ruff.lint]
|
|
77
77
|
extend-select = ["E", "F", "I", "N", "B", "SIM", "NPY", "RUF", "PTH", "PD", "FURB"]
|
|
78
|
-
ignore = ["F722", "N812", "SIM102", "SIM103", "SIM105", "SIM108", "E501", "N806", "N803", "N815", "RUF005", "SIM300", "C408", "TRY401"]
|
|
78
|
+
ignore = ["F722", "N812", "SIM102", "SIM103", "SIM105", "SIM108", "SIM114", "E501", "N806", "N803", "N815", "RUF005", "SIM300", "C408", "TRY401"]
|
|
79
79
|
|
|
80
80
|
[tool.ruff.lint.per-file-ignores]
|
|
81
81
|
"**/__init__.py" = ["F401", "F403"]
|
|
@@ -96,5 +96,4 @@ replace-imports-with-any = ["hdbscan"]
|
|
|
96
96
|
|
|
97
97
|
[tool.ty.rules]
|
|
98
98
|
"unsupported-base" = "ignore"
|
|
99
|
-
"invalid-key" = "ignore"
|
|
100
99
|
"possibly-unresolved-reference" = "error"
|
|
@@ -76,7 +76,12 @@ from .util.logging_util import (
|
|
|
76
76
|
get_logger,
|
|
77
77
|
set_log_level,
|
|
78
78
|
)
|
|
79
|
-
from .util.motion import
|
|
79
|
+
from .util.motion import (
|
|
80
|
+
MotionInfo,
|
|
81
|
+
detect_for_motion,
|
|
82
|
+
get_motion_info,
|
|
83
|
+
try_load_motion_info,
|
|
84
|
+
)
|
|
80
85
|
from .util.noise_util import EmbeddedNoise
|
|
81
86
|
from .util.preprocess_util import preprocess
|
|
82
87
|
from .util.py_util import databag, ensure_path
|
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
from typing import cast
|
|
2
2
|
|
|
3
3
|
import h5py
|
|
4
|
+
import numba
|
|
4
5
|
import numpy as np
|
|
5
6
|
from scipy.cluster.hierarchy import fcluster, linkage
|
|
6
7
|
from scipy.spatial import KDTree
|
|
7
8
|
|
|
8
|
-
from ..util import
|
|
9
|
+
from ..util import waveform_util
|
|
9
10
|
from ..util.data_util import (
|
|
10
11
|
DARTsortSorting,
|
|
11
12
|
apply_label_remapping_in_place,
|
|
13
|
+
count_not_sorted,
|
|
12
14
|
mean_by_label_1d,
|
|
13
15
|
pos_int_unique_and_counts,
|
|
16
|
+
yield_masked_chunks,
|
|
14
17
|
)
|
|
15
18
|
from ..util.logging_util import get_logger
|
|
16
19
|
from ..util.motion import MotionInfo
|
|
20
|
+
from ..util.py_util import databag
|
|
17
21
|
|
|
18
22
|
logger = get_logger(__name__)
|
|
19
23
|
|
|
@@ -509,7 +513,7 @@ def get_main_channel_pcs(
|
|
|
509
513
|
with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
|
|
510
514
|
feats_dset = h5[dataset_name]
|
|
511
515
|
channel_index = cast(h5py.Dataset, h5["channel_index"])[:]
|
|
512
|
-
for ixs, feats in
|
|
516
|
+
for ixs, feats in yield_masked_chunks(
|
|
513
517
|
mask, feats_dset, show_progress=show_progress, desc_prefix="Main channel"
|
|
514
518
|
):
|
|
515
519
|
feats = feats[:, :rank]
|
|
@@ -576,3 +580,102 @@ def decrumb(
|
|
|
576
580
|
if flatten:
|
|
577
581
|
sorting = sorting.flatten(in_place=in_place)
|
|
578
582
|
return sorting
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
@databag
|
|
586
|
+
class ViolationCounts:
|
|
587
|
+
unit_ids: np.ndarray
|
|
588
|
+
spike_counts: np.ndarray
|
|
589
|
+
"""Same shape as unit_ids (flat)"""
|
|
590
|
+
viol_counts: np.ndarray
|
|
591
|
+
"""Indexed by pair of ids (not flat)"""
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def violation_matrix(
|
|
595
|
+
st: DARTsortSorting, *, censor_ms: float = 0.25, viol_ms: float = 1.0
|
|
596
|
+
) -> ViolationCounts:
|
|
597
|
+
"""Count ACG and CCG violations within viol_ms
|
|
598
|
+
|
|
599
|
+
Times within censor_ms of each other are ignored in the violation
|
|
600
|
+
count. The censorship is right-exclusive, so that if censor_ms is 0,
|
|
601
|
+
exact duplicates are counted; if censor_ms corresponds to 10 samples,
|
|
602
|
+
9-sample viols are excluded and 10-sample viols are counted.
|
|
603
|
+
"""
|
|
604
|
+
assert st.labels is not None
|
|
605
|
+
censor_samples = int(censor_ms * (st.sampling_frequency / 1000.0))
|
|
606
|
+
viol_samples = int(viol_ms * (st.sampling_frequency / 1000.0))
|
|
607
|
+
|
|
608
|
+
unit_ids, spike_counts, _ = pos_int_unique_and_counts(st.labels)
|
|
609
|
+
nu = (unit_ids.max() + 1).item() if unit_ids.size else 0
|
|
610
|
+
if not nu or (viol_samples < max(0, censor_samples)):
|
|
611
|
+
# nothing can be counted, but keep the matrix shape consistent
|
|
612
|
+
return ViolationCounts(
|
|
613
|
+
unit_ids=unit_ids,
|
|
614
|
+
spike_counts=spike_counts,
|
|
615
|
+
viol_counts=np.zeros((nu, nu), dtype=np.int64),
|
|
616
|
+
)
|
|
617
|
+
|
|
618
|
+
labels = st.labels
|
|
619
|
+
times = st.times_samples
|
|
620
|
+
if count_not_sorted(times) > 0:
|
|
621
|
+
tsort = np.argsort(times, kind="stable")
|
|
622
|
+
labels = labels[tsort]
|
|
623
|
+
times = times[tsort]
|
|
624
|
+
|
|
625
|
+
# count in chunks, per thread buffer; counts are ti<=tj
|
|
626
|
+
n = times.size
|
|
627
|
+
nchunks = max(1, numba.get_num_threads())
|
|
628
|
+
nchunks = min(nchunks, max(n, 1))
|
|
629
|
+
starts = (np.arange(nchunks + 1) * n) // nchunks
|
|
630
|
+
viol_counts = np.zeros((nchunks, nu, nu), dtype=np.int64)
|
|
631
|
+
|
|
632
|
+
_violation_count_matrix(
|
|
633
|
+
times, labels, censor_samples, viol_samples, starts, viol_counts
|
|
634
|
+
)
|
|
635
|
+
|
|
636
|
+
viol_counts = viol_counts.sum(axis=0)
|
|
637
|
+
viol_diag = np.diagonal(viol_counts).copy()
|
|
638
|
+
viol_counts += viol_counts.T
|
|
639
|
+
np.fill_diagonal(viol_counts, viol_diag)
|
|
640
|
+
|
|
641
|
+
return ViolationCounts(
|
|
642
|
+
unit_ids=unit_ids,
|
|
643
|
+
spike_counts=spike_counts,
|
|
644
|
+
viol_counts=viol_counts,
|
|
645
|
+
)
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
@numba.njit(nogil=True, parallel=True)
|
|
649
|
+
def _violation_count_matrix(
|
|
650
|
+
times: np.ndarray,
|
|
651
|
+
labels: np.ndarray,
|
|
652
|
+
censor_samples: int,
|
|
653
|
+
viol_samples: int,
|
|
654
|
+
starts: np.ndarray,
|
|
655
|
+
counts: np.ndarray,
|
|
656
|
+
):
|
|
657
|
+
n = times.shape[0]
|
|
658
|
+
|
|
659
|
+
# parallelize over chunks
|
|
660
|
+
for c in numba.prange(starts.shape[0] - 1): # ty: ignore[not-iterable]
|
|
661
|
+
out = counts[c]
|
|
662
|
+
|
|
663
|
+
for i in range(starts[c], starts[c + 1]):
|
|
664
|
+
li = labels[i]
|
|
665
|
+
if li < 0:
|
|
666
|
+
continue
|
|
667
|
+
|
|
668
|
+
ti = times[i]
|
|
669
|
+
first = ti + censor_samples
|
|
670
|
+
last = ti + viol_samples
|
|
671
|
+
|
|
672
|
+
# make sure to read js past the chunk end!
|
|
673
|
+
for j in range(i + 1, n):
|
|
674
|
+
if times[j] < first:
|
|
675
|
+
continue
|
|
676
|
+
if times[j] > last: # be inclusive here i suppose
|
|
677
|
+
break
|
|
678
|
+
lj = labels[j]
|
|
679
|
+
if lj < 0:
|
|
680
|
+
continue
|
|
681
|
+
out[li, lj] += 1
|
|
@@ -188,7 +188,7 @@ def get_smoothed_density_ratio(
|
|
|
188
188
|
else:
|
|
189
189
|
dens_ = dens[0]
|
|
190
190
|
dens_ /= dens[1]
|
|
191
|
-
np.nan_to_num(dens_, out=dens_) # type: ignore
|
|
191
|
+
np.nan_to_num(dens_, out=dens_, posinf=np.inf, neginf=-np.inf) # type: ignore
|
|
192
192
|
return dens_
|
|
193
193
|
|
|
194
194
|
|
|
@@ -407,7 +407,7 @@ def kdt_density(
|
|
|
407
407
|
desc=f"KDTdens[{n_jobs}]",
|
|
408
408
|
):
|
|
409
409
|
density[i0:i1] = dens
|
|
410
|
-
np.nan_to_num(density, copy=False)
|
|
410
|
+
np.nan_to_num(density, posinf=np.inf, neginf=-np.inf, copy=False)
|
|
411
411
|
return density
|
|
412
412
|
|
|
413
413
|
|
|
@@ -358,21 +358,47 @@ def get_deconv_resid_decrease_iter(
|
|
|
358
358
|
)
|
|
359
359
|
|
|
360
360
|
|
|
361
|
-
def combine_templates(template_data_a, template_data_b):
|
|
361
|
+
def combine_templates(template_data_a: TemplateData, template_data_b: TemplateData):
|
|
362
362
|
rgeom = template_data_a.registered_geom
|
|
363
363
|
if rgeom is not None:
|
|
364
|
+
assert template_data_b.registered_geom is not None
|
|
364
365
|
if not np.array_equal(rgeom, template_data_b.registered_geom):
|
|
365
366
|
raise ValueError(
|
|
366
367
|
f"Template data had different registered geoms: "
|
|
367
368
|
f"{template_data_a.registered_geom=} {template_data_b.registered_geom=}"
|
|
368
369
|
)
|
|
369
370
|
|
|
371
|
+
ta = template_data_a.templates
|
|
372
|
+
tb = template_data_b.templates
|
|
373
|
+
assert ta.shape[2] == tb.shape[2]
|
|
374
|
+
if ta.shape[1] > tb.shape[1]:
|
|
375
|
+
i0 = (
|
|
376
|
+
template_data_a.trough_offset_samples
|
|
377
|
+
- template_data_b.trough_offset_samples
|
|
378
|
+
)
|
|
379
|
+
assert i0 >= 0
|
|
380
|
+
i1 = i0 + template_data_b.spike_length_samples
|
|
381
|
+
ta = ta[:, i0:i1]
|
|
382
|
+
trough_offset_samples = template_data_b.trough_offset_samples
|
|
383
|
+
sampling_frequency = template_data_b.sampling_frequency
|
|
384
|
+
elif tb.shape[1] > ta.shape[1]:
|
|
385
|
+
i0 = (
|
|
386
|
+
template_data_b.trough_offset_samples
|
|
387
|
+
- template_data_a.trough_offset_samples
|
|
388
|
+
)
|
|
389
|
+
assert i0 >= 0
|
|
390
|
+
i1 = i0 + template_data_a.spike_length_samples
|
|
391
|
+
tb = tb[:, i0:i1]
|
|
392
|
+
trough_offset_samples = template_data_a.trough_offset_samples
|
|
393
|
+
sampling_frequency = template_data_a.sampling_frequency
|
|
394
|
+
else:
|
|
395
|
+
trough_offset_samples = template_data_a.trough_offset_samples
|
|
396
|
+
sampling_frequency = template_data_a.sampling_frequency
|
|
397
|
+
|
|
370
398
|
ids_a = template_data_a.unit_ids
|
|
371
399
|
ids_b = template_data_b.unit_ids + ids_a.max() + 1
|
|
372
400
|
unit_ids = np.concatenate((ids_a, ids_b))
|
|
373
|
-
templates = np.concatenate(
|
|
374
|
-
(template_data_a.templates, template_data_b.templates), axis=0
|
|
375
|
-
)
|
|
401
|
+
templates = np.concatenate((ta, tb), axis=0)
|
|
376
402
|
spike_counts = np.concatenate(
|
|
377
403
|
(template_data_a.spike_counts, template_data_b.spike_counts)
|
|
378
404
|
)
|
|
@@ -389,8 +415,8 @@ def combine_templates(template_data_a, template_data_b):
|
|
|
389
415
|
spike_counts=spike_counts,
|
|
390
416
|
registered_geom=rgeom,
|
|
391
417
|
spike_counts_by_channel=spike_counts_by_channel,
|
|
392
|
-
trough_offset_samples=
|
|
393
|
-
sampling_frequency=
|
|
418
|
+
trough_offset_samples=trough_offset_samples,
|
|
419
|
+
sampling_frequency=sampling_frequency,
|
|
394
420
|
)
|
|
395
421
|
|
|
396
422
|
cross_mask = np.logical_and(
|
|
@@ -5855,31 +5855,40 @@ def _noise_factors(*, noise, obs_ix, miss_near_ix, cache_prefix):
|
|
|
5855
5855
|
jmnixv = jmnix[jmnixvix]
|
|
5856
5856
|
ncoi = joixv.numel()
|
|
5857
5857
|
ncmi = jmnixv.numel()
|
|
5858
|
+
assert ncoi > 0
|
|
5858
5859
|
|
|
5860
|
+
# -- observed-only stuff
|
|
5861
|
+
# calculate observed-only stuff
|
|
5859
5862
|
jCoo = noise.marginal_covariance(
|
|
5860
5863
|
channels=joixv, cache_prefix=cache_prefix, cache_key=j
|
|
5861
5864
|
)
|
|
5862
|
-
jCom = noise.offdiag_covariance(
|
|
5863
|
-
channels_left=joixv, channels_right=jmnixv, device=dev
|
|
5864
|
-
)
|
|
5865
|
-
jCom = jCom.to_dense().to(device=dev)
|
|
5866
|
-
|
|
5867
5865
|
jL = jCoo.cholesky(upper=False) # C = LL'
|
|
5868
5866
|
jLinv = jL.inverse().to_dense()
|
|
5869
5867
|
jCooinv = jLinv.T @ jLinv # Cinv = Linv' Linv
|
|
5870
5868
|
jlogdet = 2.0 * jL.to_dense().diagonal(dim1=-2, dim2=-1).log().sum()
|
|
5871
|
-
jCooinvCom = jCooinv @ jCom
|
|
5872
5869
|
|
|
5870
|
+
# set observed-only stuff
|
|
5873
5871
|
logdet[j] = jlogdet
|
|
5874
5872
|
# fancy inds to front! love that.
|
|
5875
|
-
|
|
5876
|
-
Cooinv[j, :, joixvix[:, None], :, joixvix[None]] =
|
|
5873
|
+
jCooinv_out = jCooinv.view(rank, ncoi, rank, ncoi).permute(1, 3, 0, 2)
|
|
5874
|
+
Cooinv[j, :, joixvix[:, None], :, joixvix[None]] = jCooinv_out
|
|
5875
|
+
jLinv_out = jLinv.view(rank, ncoi, rank, ncoi).permute(1, 3, 0, 2)
|
|
5876
|
+
Linv[j, :, joixvix[:, None], :, joixvix[None]] = jLinv_out
|
|
5877
|
+
|
|
5878
|
+
# -- observed-missing stuff
|
|
5879
|
+
if ncmi == 0:
|
|
5880
|
+
# i am a lonely island
|
|
5881
|
+
# importantly, that means my Com is 0. so everything is 0 below.
|
|
5882
|
+
continue
|
|
5883
|
+
jCom = noise.offdiag_covariance(
|
|
5884
|
+
channels_left=joixv, channels_right=jmnixv, device=dev
|
|
5885
|
+
)
|
|
5886
|
+
jCom = jCom.to_dense().to(device=dev)
|
|
5887
|
+
jCooinvCom = jCooinv @ jCom
|
|
5877
5888
|
jCooinvCom = jCooinvCom.view(rank, ncoi, rank, ncmi)
|
|
5878
5889
|
CooinvCom[j, :, joixvix[:, None], :, jmnixvix[None]] = jCooinvCom.permute(
|
|
5879
5890
|
1, 3, 0, 2
|
|
5880
5891
|
)
|
|
5881
|
-
jLinv = jLinv.view(rank, ncoi, rank, ncoi)
|
|
5882
|
-
Linv[j, :, joixvix[:, None], :, joixvix[None]] = jLinv.permute(1, 3, 0, 2)
|
|
5883
5892
|
|
|
5884
5893
|
obsdim = rank * nc_obs
|
|
5885
5894
|
missdim = rank * nc_miss_near
|
|
@@ -6739,7 +6748,7 @@ def _update_lut_mean_batch(
|
|
|
6739
6748
|
|
|
6740
6749
|
# constplogdet. add in the signal-rank-0-only terms.
|
|
6741
6750
|
lut_params.constplogdet[i0:i1] = neighb_cov.nobs[nn].mul_(LOG_2PI) # type: ignore # ty: ignore[x]
|
|
6742
|
-
lut_params.constplogdet[i0:i1] += neighb_cov.b.logdet[nn]
|
|
6751
|
+
lut_params.constplogdet[i0:i1] += neighb_cov.b.logdet[nn] # ty: ignore[not-subscriptable]
|
|
6743
6752
|
if pnoid:
|
|
6744
6753
|
assert lut_params.constplogdet[i0:i1].isfinite().all() # type: ignore # ty: ignore[x]
|
|
6745
6754
|
|
|
@@ -6799,7 +6808,7 @@ def _update_lut_ppca_batch(
|
|
|
6799
6808
|
assert lut_params.Tpad is not None
|
|
6800
6809
|
lut_params.Tpad[i0:i1, :, :-1] = T # type: ignore # ty: ignore[x]
|
|
6801
6810
|
cap_logdet = L.diagonal(dim1=-2, dim2=-1).log().sum(dim=1).mul_(2.0)
|
|
6802
|
-
lut_params.constplogdet[i0:i1] += cap_logdet
|
|
6811
|
+
lut_params.constplogdet[i0:i1] += cap_logdet # ty: ignore[not-subscriptable]
|
|
6803
6812
|
if pnoid:
|
|
6804
6813
|
assert lut_params.b.constplogdet[i0:i1].isfinite().all()
|
|
6805
6814
|
|
|
@@ -205,6 +205,7 @@ class DARTsortUserConfig:
|
|
|
205
205
|
"""Upsampling of templates during matching to allow for temporal aliasing of waveforms."""
|
|
206
206
|
|
|
207
207
|
# -- final merge step
|
|
208
|
+
#TODO name this more prominently, clarify flags for dedup ms, decouple dedup and agg
|
|
208
209
|
agg_kind: Literal["none", "template_distance", "qda"] = "qda"
|
|
209
210
|
"""Final distance or GMM-based merge type."""
|
|
210
211
|
|
|
@@ -39,11 +39,12 @@ from ..util.internal_config import (
|
|
|
39
39
|
)
|
|
40
40
|
from ..util.interpolation_util import StableFeaturesInterpolator, pad_geom
|
|
41
41
|
from ..util.motion import MotionInfo
|
|
42
|
-
from ..util.py_util import databag
|
|
42
|
+
from ..util.py_util import databag, panic
|
|
43
43
|
from ..util.spikeio import read_waveforms_channel_index
|
|
44
44
|
from ..util.waveform_util import make_channel_index
|
|
45
45
|
|
|
46
46
|
logger = logging_util.get_logger(__name__)
|
|
47
|
+
vis_distance_merge_cfg = TemplateMergeConfig(min_spatial_cosine=0.8)
|
|
47
48
|
|
|
48
49
|
|
|
49
50
|
@dataclass
|
|
@@ -63,6 +64,7 @@ class DARTsortAnalysis:
|
|
|
63
64
|
registered_geom: np.ndarray
|
|
64
65
|
extract_channel_index: np.ndarray | None
|
|
65
66
|
vis_channel_index: np.ndarray
|
|
67
|
+
read_channel_index: np.ndarray
|
|
66
68
|
xyza: np.ndarray | None
|
|
67
69
|
x: np.ndarray | None
|
|
68
70
|
z: np.ndarray | None
|
|
@@ -94,9 +96,7 @@ class DARTsortAnalysis:
|
|
|
94
96
|
name: str | None = None,
|
|
95
97
|
template_data: TemplateData | None = None,
|
|
96
98
|
template_cfg: TemplateConfig | None = raw_template_cfg,
|
|
97
|
-
template_merge_cfg: TemplateMergeConfig =
|
|
98
|
-
min_spatial_cosine=0.8
|
|
99
|
-
),
|
|
99
|
+
template_merge_cfg: TemplateMergeConfig = vis_distance_merge_cfg,
|
|
100
100
|
clustering_features_cfg: ClusteringFeaturesConfig = default_clustering_features_cfg,
|
|
101
101
|
computation_cfg: ComputationConfig | None = None,
|
|
102
102
|
allow_qda: bool = True,
|
|
@@ -136,7 +136,7 @@ class DARTsortAnalysis:
|
|
|
136
136
|
tpca = get_tpca(
|
|
137
137
|
sorting, featurization_pipeline_pt=featurization_pipeline_pt
|
|
138
138
|
)
|
|
139
|
-
except ValueError:
|
|
139
|
+
except (ValueError, FileNotFoundError):
|
|
140
140
|
tpca = None
|
|
141
141
|
else:
|
|
142
142
|
tpca = None
|
|
@@ -272,13 +272,19 @@ class DARTsortAnalysis:
|
|
|
272
272
|
erp = None
|
|
273
273
|
stable_vis_channels = None
|
|
274
274
|
|
|
275
|
+
orig_geom_vis_channel_index = make_channel_index(
|
|
276
|
+
geom=motion.geom,
|
|
277
|
+
radius=vis_radius,
|
|
278
|
+
p=vis_neighborhood_p,
|
|
279
|
+
to_torch=False,
|
|
280
|
+
)
|
|
275
281
|
if channel_index is None:
|
|
276
|
-
channel_index =
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
+
channel_index = orig_geom_vis_channel_index
|
|
283
|
+
|
|
284
|
+
if channel_index.shape[1] < orig_geom_vis_channel_index.shape[1]:
|
|
285
|
+
read_channel_index = orig_geom_vis_channel_index
|
|
286
|
+
else:
|
|
287
|
+
read_channel_index = channel_index
|
|
282
288
|
|
|
283
289
|
unit_ids, spike_counts = np.unique(sorting.labels, return_counts=True) # type: ignore
|
|
284
290
|
spike_counts = spike_counts[unit_ids >= 0]
|
|
@@ -310,6 +316,7 @@ class DARTsortAnalysis:
|
|
|
310
316
|
registered_geom=motion.rgeom,
|
|
311
317
|
extract_channel_index=channel_index,
|
|
312
318
|
vis_channel_index=vis_channel_index,
|
|
319
|
+
read_channel_index=read_channel_index,
|
|
313
320
|
probe_disp=probe_disp,
|
|
314
321
|
stable_vis_channels=stable_vis_channels,
|
|
315
322
|
xyza=xyza,
|
|
@@ -355,8 +362,8 @@ class DARTsortAnalysis:
|
|
|
355
362
|
if unit_id is not None:
|
|
356
363
|
return np.ptp(self.template_data.unit_templates(unit_id))
|
|
357
364
|
amplitudes = np.zeros(self.sorting.unit_ids.shape)
|
|
358
|
-
for j,
|
|
359
|
-
temps = self.template_data.unit_templates(
|
|
365
|
+
for j, u in enumerate(self.sorting.unit_ids):
|
|
366
|
+
temps = self.template_data.unit_templates(u)
|
|
360
367
|
amplitudes[j] = np.ptp(np.nan_to_num(temps))
|
|
361
368
|
return amplitudes
|
|
362
369
|
|
|
@@ -413,7 +420,7 @@ class DARTsortAnalysis:
|
|
|
413
420
|
elif unit_id is not None:
|
|
414
421
|
which = self.in_unit(unit_id)
|
|
415
422
|
else:
|
|
416
|
-
|
|
423
|
+
panic()
|
|
417
424
|
|
|
418
425
|
if max_count is not None and which.size > max_count:
|
|
419
426
|
rg = np.random.default_rng(random_seed)
|
|
@@ -430,14 +437,10 @@ class DARTsortAnalysis:
|
|
|
430
437
|
|
|
431
438
|
# read waveforms from disk
|
|
432
439
|
read_chans = self.sorting.channels[which]
|
|
433
|
-
if self.extract_channel_index is None:
|
|
434
|
-
read_channel_index = self.vis_channel_index
|
|
435
|
-
else:
|
|
436
|
-
read_channel_index = self.extract_channel_index
|
|
437
440
|
waveforms = read_waveforms_channel_index(
|
|
438
441
|
recording=self.recording,
|
|
439
442
|
times_samples=self.sorting.times_samples[which],
|
|
440
|
-
channel_index=read_channel_index,
|
|
443
|
+
channel_index=self.read_channel_index,
|
|
441
444
|
main_channels=read_chans,
|
|
442
445
|
trough_offset_samples=self.trough_offset_samples,
|
|
443
446
|
spike_length_samples=self.spike_length_samples,
|
|
@@ -458,7 +461,7 @@ class DARTsortAnalysis:
|
|
|
458
461
|
channels[None], (waveforms.shape[0], waveforms.shape[2])
|
|
459
462
|
)
|
|
460
463
|
else:
|
|
461
|
-
channels = read_channel_index[read_chans]
|
|
464
|
+
channels = self.read_channel_index[read_chans]
|
|
462
465
|
if main_channel is None:
|
|
463
466
|
main_channel = self.unit_max_channel(unit_id)
|
|
464
467
|
return WaveformsBag(
|
|
@@ -507,7 +510,7 @@ class DARTsortAnalysis:
|
|
|
507
510
|
elif unit_id is not None:
|
|
508
511
|
which = self.in_unit(unit_id)
|
|
509
512
|
else:
|
|
510
|
-
|
|
513
|
+
panic()
|
|
511
514
|
|
|
512
515
|
if max_count is not None and which.size > max_count:
|
|
513
516
|
rg = np.random.default_rng(random_seed)
|
|
@@ -660,17 +663,12 @@ class DARTsortAnalysis:
|
|
|
660
663
|
else:
|
|
661
664
|
n_pitches_shift = None
|
|
662
665
|
|
|
663
|
-
if self.extract_channel_index is None:
|
|
664
|
-
read_channel_index = self.vis_channel_index
|
|
665
|
-
else:
|
|
666
|
-
read_channel_index = self.extract_channel_index
|
|
667
|
-
|
|
668
666
|
waveforms_valid = get_waveforms_on_static_channels(
|
|
669
667
|
waveforms=waveforms,
|
|
670
668
|
geom=self.geom,
|
|
671
669
|
n_pitches_shift=n_pitches_shift,
|
|
672
670
|
main_channels=read_chans,
|
|
673
|
-
channel_index=read_channel_index,
|
|
671
|
+
channel_index=self.read_channel_index,
|
|
674
672
|
target_channels=show_chans,
|
|
675
673
|
registered_geom=self.registered_geom,
|
|
676
674
|
)
|
|
@@ -59,6 +59,9 @@ class DARTsortGroundTruthComparison:
|
|
|
59
59
|
self.delta_frames = self.comparison.delta_frames
|
|
60
60
|
self.has_templates = self.tested_analysis.template_data is not None
|
|
61
61
|
self._agreement_scores = None
|
|
62
|
+
assert np.array_equal(
|
|
63
|
+
self.gt_analysis.vis_channel_index, self.tested_analysis.vis_channel_index
|
|
64
|
+
)
|
|
62
65
|
|
|
63
66
|
if self.compute_distances and self.has_templates:
|
|
64
67
|
if self.verbose:
|
|
@@ -259,7 +262,9 @@ class DARTsortGroundTruthComparison:
|
|
|
259
262
|
except ValueError:
|
|
260
263
|
pass
|
|
261
264
|
if self.has_templates and (force_distances or self.compute_distances):
|
|
262
|
-
dist = np.nan_to_num(
|
|
265
|
+
dist = np.nan_to_num(
|
|
266
|
+
self.template_distances, nan=np.inf, posinf=np.inf, neginf=-np.inf
|
|
267
|
+
).min(axis=1)
|
|
263
268
|
df["min_temp_dist"] = dist
|
|
264
269
|
rec = []
|
|
265
270
|
for uid in df.index:
|
|
@@ -296,8 +301,8 @@ class DARTsortGroundTruthComparison:
|
|
|
296
301
|
def relevant_tested_units(self, gt_unit_id: int | None = None, threshold=0.05):
|
|
297
302
|
if gt_unit_id is None:
|
|
298
303
|
tested_units = set()
|
|
299
|
-
for
|
|
300
|
-
tested_units.update(self.relevant_tested_units(gt_unit_id=
|
|
304
|
+
for uu in self.gt_analysis.unit_ids:
|
|
305
|
+
tested_units.update(self.relevant_tested_units(gt_unit_id=uu))
|
|
301
306
|
return tested_units
|
|
302
307
|
ag = self.agreement_scores.loc[gt_unit_id]
|
|
303
308
|
return ag.index[ag.array >= threshold]
|
|
@@ -334,7 +339,9 @@ class DARTsortGroundTruthComparison:
|
|
|
334
339
|
if udt.size:
|
|
335
340
|
match_dt_rms[j] = np.sqrt(np.square(udt).mean())
|
|
336
341
|
except ValueError as e:
|
|
337
|
-
warnings.warn(
|
|
342
|
+
warnings.warn(
|
|
343
|
+
f"ValueError in misalignment. SI matching bug. {e=}", stacklevel=2
|
|
344
|
+
)
|
|
338
345
|
return match_dt_rms
|
|
339
346
|
|
|
340
347
|
def matched_misalignment(self, gt_unit_id):
|
|
@@ -477,7 +484,8 @@ class DARTsortGroundTruthComparison:
|
|
|
477
484
|
warnings.warn(
|
|
478
485
|
f"Strange match sizes for {gt_unit=} {tested_unit=}: "
|
|
479
486
|
f"{matched_gt_indices.shape=} {matched_tested_indices.shape=} "
|
|
480
|
-
f"{matched_tested_mask.sum()=}",
|
|
487
|
+
f"{matched_tested_mask.sum()=}",
|
|
488
|
+
stacklevel=2,
|
|
481
489
|
)
|
|
482
490
|
else:
|
|
483
491
|
matched_tested_indices = np.zeros(shape=(0,), dtype=np.int64)
|
|
@@ -596,7 +604,7 @@ class DARTsortGroundTruthComparison:
|
|
|
596
604
|
gt_times_ms = self.gt_analysis.sorting.times_samples[in_gt] * to_ms
|
|
597
605
|
tested_times_ms = self.tested_analysis.sorting.times_samples[in_tu] * to_ms
|
|
598
606
|
gt_kdt = KDTree(gt_times_ms[:, None])
|
|
599
|
-
|
|
607
|
+
_dd, ii = gt_kdt.query(
|
|
600
608
|
tested_times_ms[:, None], distance_upper_bound=self.delta_time
|
|
601
609
|
)
|
|
602
610
|
ii = np.atleast_1d(ii)
|
|
@@ -620,7 +628,10 @@ class DARTsortGroundTruthComparison:
|
|
|
620
628
|
# waveforms are read at GT unit max channel
|
|
621
629
|
gt_max_chan = self.gt_analysis.unit_max_channel(gt_unit)
|
|
622
630
|
waveform_kw = dict(
|
|
623
|
-
max_count=max_samples_per_category,
|
|
631
|
+
max_count=max_samples_per_category,
|
|
632
|
+
random_seed=rg,
|
|
633
|
+
main_channel=gt_max_chan,
|
|
634
|
+
to_main_channel=True,
|
|
624
635
|
)
|
|
625
636
|
|
|
626
637
|
# return vars dict. lots of stuff going in here.
|
|
@@ -671,7 +682,6 @@ class DARTsortGroundTruthComparison:
|
|
|
671
682
|
unit_id=tested_unit,
|
|
672
683
|
which=ind_groups["only_tested_indices"],
|
|
673
684
|
**waveform_kw, # type: ignore
|
|
674
|
-
to_main_channel=True,
|
|
675
685
|
)
|
|
676
686
|
else:
|
|
677
687
|
fp_waves = None
|
|
@@ -741,7 +751,8 @@ class DARTsortGTVersus:
|
|
|
741
751
|
self.other_analyses = other_analyses
|
|
742
752
|
self.other_names = [oa.name for oa in other_analyses]
|
|
743
753
|
self.other_templates = [
|
|
744
|
-
(oa.name or f"Test{c}")
|
|
754
|
+
(oa.name or f"Test{c}")
|
|
755
|
+
for oa, c in zip(other_analyses, self.default_ids, strict=True)
|
|
745
756
|
]
|
|
746
757
|
self.other_sortings = [oa.sorting for oa in other_analyses]
|
|
747
758
|
|
|
@@ -751,7 +762,7 @@ class DARTsortGTVersus:
|
|
|
751
762
|
assert len(comparisons) == self.n_vs
|
|
752
763
|
|
|
753
764
|
self.cmps = []
|
|
754
|
-
for cmp, oa in zip(comparisons, other_analyses):
|
|
765
|
+
for cmp, oa in zip(comparisons, other_analyses, strict=True):
|
|
755
766
|
if cmp is not None:
|
|
756
767
|
assert cmp.tested_analysis.name == oa.name
|
|
757
768
|
else:
|
|
@@ -773,7 +784,7 @@ class DARTsortGTVersus:
|
|
|
773
784
|
if self._unit_vs_df is not None:
|
|
774
785
|
return self._unit_vs_df.copy(deep=True)
|
|
775
786
|
dfs = [ocmp.unit_info_dataframe().reset_index() for ocmp in self.cmps]
|
|
776
|
-
for df, sorter in zip(dfs, self.other_names):
|
|
787
|
+
for df, sorter in zip(dfs, self.other_names, strict=True):
|
|
777
788
|
df[self.sorter_var] = sorter
|
|
778
789
|
self._unit_vs_df = pd.concat(dfs, axis=0)
|
|
779
790
|
return self._unit_vs_df.copy(deep=True)
|