DARTsort 0.5.21__tar.gz → 0.5.22__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.21 → dartsort-0.5.22}/.gitignore +1 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/PKG-INFO +1 -1
- {dartsort-0.5.21 → dartsort-0.5.22}/pyproject.toml +1 -1
- dartsort-0.5.22/src/dartsort/__main__.py +4 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/cli.py +8 -4
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/clustering/agglomerate.py +33 -19
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/clustering/cluster_util.py +56 -47
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/clustering/clustering.py +3 -2
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/clustering/clustering_features.py +82 -103
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/clustering/density.py +4 -4
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/clustering/mixture.py +4 -10
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/clustering/refine_util.py +1 -4
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/config.py +1 -1
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/evaluate/analysis.py +2 -2
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/evaluate/comparison.py +14 -14
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/evaluate/noise_recording_tools.py +1 -1
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/evaluate/sim_template_tools.py +179 -84
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/main.py +45 -14
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/matching.py +0 -6
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/templates/postprocess_util.py +8 -8
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/templates/templates.py +0 -2
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/amplitudes.py +3 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/mixture_classifier.py +2 -1
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/pipeline.py +6 -5
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/reduction.py +3 -6
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/transform_base.py +5 -2
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/cli_util.py +2 -2
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/data_util.py +309 -69
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/internal_config.py +8 -4
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/interpolation_util.py +18 -14
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/main_util.py +9 -2
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/motion.py +11 -3
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/multiprocessing_util.py +1 -1
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/noise_util.py +2 -2
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/peel_util.py +11 -15
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort.egg-info/SOURCES.txt +1 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_alignment.py +3 -3
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_cluster_util.py +2 -2
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_clustering.py +3 -2
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_matching.py +1 -1
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_simkit.py +92 -17
- {dartsort-0.5.21 → dartsort-0.5.22}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/README.md +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/docs/index.md +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/docs/main_api.md +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/environment.yml +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/notebook/.keep +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/notebook/ibl_dandi_example.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/setup.cfg +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/clustering/kmeans.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/evaluate/simkit.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/peel_base.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/peel_lib.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/subtract.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/decollider.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/spiketorch.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/analysis_plots.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/mixture.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/vismain.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/conftest.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_agglomerate.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_config.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_data_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_density.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_detect.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_linalg.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_modeling.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_subtract.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_templates.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_threshold.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_transform.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.21 → dartsort-0.5.22}/zensical.toml +0 -0
|
@@ -75,7 +75,7 @@ line-length = 88
|
|
|
75
75
|
|
|
76
76
|
[tool.ruff.lint]
|
|
77
77
|
extend-select = ["E", "F", "I", "N", "B", "SIM", "NPY", "RUF", "PTH", "PD", "FURB"]
|
|
78
|
-
ignore = ["F722", "N812", "SIM102", "SIM108", "E501", "N806", "N803", "N815", "RUF005", "SIM300", "C408", "TRY401"]
|
|
78
|
+
ignore = ["F722", "N812", "SIM102", "SIM103", "SIM105", "SIM108", "E501", "N806", "N803", "N815", "RUF005", "SIM300", "C408", "TRY401"]
|
|
79
79
|
|
|
80
80
|
[tool.ruff.lint.per-file-ignores]
|
|
81
81
|
"**/__init__.py" = ["F401", "F403"]
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import logging
|
|
3
|
+
import pickle
|
|
3
4
|
import sys
|
|
4
5
|
|
|
5
6
|
import spikeinterface.core as sc
|
|
@@ -13,9 +14,7 @@ def dartsort_cli():
|
|
|
13
14
|
|
|
14
15
|
Try `dartsort --help` to start.
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
I am figuring out how to do preprocessing still. It may be configured?
|
|
17
|
+
---
|
|
19
18
|
"""
|
|
20
19
|
# -- define CLI
|
|
21
20
|
assert dartsort_cli.__doc__ is not None
|
|
@@ -90,7 +89,12 @@ def dartsort_cli():
|
|
|
90
89
|
|
|
91
90
|
# load the recording
|
|
92
91
|
try:
|
|
93
|
-
|
|
92
|
+
rec_path = cli_util.ensurepath(args.recording, strict=True)
|
|
93
|
+
if rec_path.suffix == ".pkl":
|
|
94
|
+
with rec_path.open("rb") as jar:
|
|
95
|
+
rec = pickle.load(jar)
|
|
96
|
+
else:
|
|
97
|
+
rec = sc.load(rec_path)
|
|
94
98
|
assert isinstance(rec, sc.BaseRecording)
|
|
95
99
|
except FileNotFoundError as e:
|
|
96
100
|
ee = FileNotFoundError(
|
|
@@ -11,7 +11,11 @@ from spikeinterface.core import BaseRecording
|
|
|
11
11
|
|
|
12
12
|
from ..templates.template_util import shared_basis_compress_templates
|
|
13
13
|
from ..templates.templates import TemplateData
|
|
14
|
-
from ..util.data_util import
|
|
14
|
+
from ..util.data_util import (
|
|
15
|
+
DARTsortSorting,
|
|
16
|
+
count_not_sorted,
|
|
17
|
+
pos_int_unique_and_counts,
|
|
18
|
+
)
|
|
15
19
|
from ..util.internal_config import (
|
|
16
20
|
ComputationConfig,
|
|
17
21
|
RefinementConfig,
|
|
@@ -72,7 +76,7 @@ def agglomerate(
|
|
|
72
76
|
|
|
73
77
|
if template_data is None:
|
|
74
78
|
did_flatten = True
|
|
75
|
-
sorting = sorting.flatten(include_gmm_properties=True)
|
|
79
|
+
sorting = sorting.flatten(include_gmm_properties=True, in_place=True)
|
|
76
80
|
else:
|
|
77
81
|
did_flatten = False
|
|
78
82
|
|
|
@@ -108,7 +112,7 @@ def agglomerate(
|
|
|
108
112
|
link=template_merge_cfg.linkage,
|
|
109
113
|
)
|
|
110
114
|
elif not did_flatten:
|
|
111
|
-
agg_sorting = sorting.flatten(include_gmm_properties=True)
|
|
115
|
+
agg_sorting = sorting.flatten(include_gmm_properties=True, in_place=True)
|
|
112
116
|
new_ids = None
|
|
113
117
|
else:
|
|
114
118
|
agg_sorting = sorting
|
|
@@ -117,7 +121,9 @@ def agglomerate(
|
|
|
117
121
|
if refinement_cfg is not None:
|
|
118
122
|
agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
|
|
119
123
|
|
|
120
|
-
agg_sorting, reorder = reorder_by_depth(
|
|
124
|
+
agg_sorting, reorder = reorder_by_depth(
|
|
125
|
+
agg_sorting, motion=motion, in_place=True
|
|
126
|
+
)
|
|
121
127
|
if new_ids is None:
|
|
122
128
|
new_ids = reorder
|
|
123
129
|
else:
|
|
@@ -241,7 +247,7 @@ def agglomerate(
|
|
|
241
247
|
|
|
242
248
|
agg_sorting = deduplicate_spikes(agg_sorting, refinement_cfg.dedup_ms)
|
|
243
249
|
|
|
244
|
-
agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion)
|
|
250
|
+
agg_sorting, reorder = reorder_by_depth(agg_sorting, motion=motion, in_place=True)
|
|
245
251
|
new_ids = reorder[np.unique(new_ids, return_inverse=True)[1]]
|
|
246
252
|
|
|
247
253
|
return Agglomeration(
|
|
@@ -862,7 +868,7 @@ def combine_gmm_scores(
|
|
|
862
868
|
)
|
|
863
869
|
|
|
864
870
|
|
|
865
|
-
@numba.njit(parallel=True)
|
|
871
|
+
@numba.njit(parallel=True, nogil=True)
|
|
866
872
|
def _combine_loop(
|
|
867
873
|
cand: np.ndarray,
|
|
868
874
|
new_id_counts: np.ndarray,
|
|
@@ -932,7 +938,10 @@ def deduplicate_spikes(
|
|
|
932
938
|
new_labels = sorting.labels.copy()
|
|
933
939
|
scores = None
|
|
934
940
|
for sck in score_by:
|
|
935
|
-
|
|
941
|
+
if not sorting.has_dataset(sck):
|
|
942
|
+
continue
|
|
943
|
+
sl = (slice(None), 0) if sck.endswith("log_liks") else ()
|
|
944
|
+
scores = sorting.load_dataset(sck, sl=sl)
|
|
936
945
|
if scores is not None:
|
|
937
946
|
logger.dartsortdebug(f"deduplicate by score {sck}")
|
|
938
947
|
break
|
|
@@ -944,33 +953,38 @@ def deduplicate_spikes(
|
|
|
944
953
|
assert scores.shape == new_labels.shape
|
|
945
954
|
|
|
946
955
|
# handle unsorted times
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
956
|
+
if count_not_sorted(sorting.times_samples) > 0:
|
|
957
|
+
tsort = np.argsort(sorting.times_samples, kind="stable")
|
|
958
|
+
new_labels = new_labels[tsort]
|
|
959
|
+
times_samples = sorting.times_samples[tsort]
|
|
960
|
+
scores = scores[tsort]
|
|
961
|
+
else:
|
|
962
|
+
times_samples = sorting.times_samples
|
|
963
|
+
tsort = None
|
|
951
964
|
|
|
952
|
-
unit_ids =
|
|
953
|
-
unit_ids = unit_ids[unit_ids >= 0]
|
|
965
|
+
unit_ids, _, _ = pos_int_unique_and_counts(new_labels)
|
|
954
966
|
ndrop = 0
|
|
967
|
+
unit_mask_tmp = np.empty(new_labels.shape, dtype=bool)
|
|
955
968
|
for unit_id in unit_ids:
|
|
956
|
-
in_unit = np.flatnonzero(new_labels
|
|
969
|
+
in_unit = np.flatnonzero(np.equal(new_labels, unit_id, out=unit_mask_tmp))
|
|
957
970
|
if in_unit.size <= 1:
|
|
958
971
|
continue
|
|
959
972
|
t = times_samples[in_unit]
|
|
960
973
|
dt = np.diff(t)
|
|
961
974
|
if dt.min() > radius_samples:
|
|
962
975
|
continue
|
|
963
|
-
discard =
|
|
976
|
+
discard = dedup_unit(t, dt, scores[in_unit], radius_samples)
|
|
964
977
|
ndrop += discard.sum()
|
|
965
978
|
new_labels[in_unit[discard]] = -1
|
|
966
979
|
|
|
967
980
|
logger.dartsortdebug(f"drop {ndrop}/{len(sorting)} isi violator spikes")
|
|
968
|
-
|
|
981
|
+
if tsort is not None:
|
|
982
|
+
new_labels = new_labels[np.argsort(tsort)]
|
|
969
983
|
|
|
970
984
|
return sorting.ephemeral_replace(labels=new_labels)
|
|
971
985
|
|
|
972
986
|
|
|
973
|
-
def
|
|
987
|
+
def dedup_unit(
|
|
974
988
|
t: np.ndarray, dt: np.ndarray, scores: np.ndarray, radius: int
|
|
975
989
|
) -> np.ndarray:
|
|
976
990
|
"""Deduplicate a single unit's spike train."""
|
|
@@ -979,7 +993,7 @@ def _dedup_unit(
|
|
|
979
993
|
return discard
|
|
980
994
|
|
|
981
995
|
|
|
982
|
-
@numba.njit
|
|
996
|
+
@numba.njit(nogil=True)
|
|
983
997
|
def _dedup_unit_loop(
|
|
984
998
|
dt: np.ndarray, scores: np.ndarray, radius: int, discard: np.ndarray
|
|
985
999
|
):
|
|
@@ -1129,7 +1143,7 @@ def coentropy(
|
|
|
1129
1143
|
)
|
|
1130
1144
|
|
|
1131
1145
|
|
|
1132
|
-
@numba.njit(parallel=True)
|
|
1146
|
+
@numba.njit(parallel=True, nogil=True)
|
|
1133
1147
|
def _calc_coentropy(
|
|
1134
1148
|
coentropy: np.ndarray,
|
|
1135
1149
|
cooccurrence: np.ndarray,
|
|
@@ -6,7 +6,12 @@ from scipy.cluster.hierarchy import fcluster, linkage
|
|
|
6
6
|
from scipy.spatial import KDTree
|
|
7
7
|
|
|
8
8
|
from ..util import data_util, waveform_util
|
|
9
|
-
from ..util.data_util import
|
|
9
|
+
from ..util.data_util import (
|
|
10
|
+
DARTsortSorting,
|
|
11
|
+
apply_label_remapping_in_place,
|
|
12
|
+
mean_by_label_1d,
|
|
13
|
+
pos_int_unique_and_counts,
|
|
14
|
+
)
|
|
10
15
|
from ..util.logging_util import get_logger
|
|
11
16
|
from ..util.motion import MotionInfo
|
|
12
17
|
|
|
@@ -59,9 +64,8 @@ def apply_reclustering(
|
|
|
59
64
|
assert sorting.labels is not None
|
|
60
65
|
|
|
61
66
|
if new_labels is None:
|
|
62
|
-
new_labels =
|
|
63
|
-
|
|
64
|
-
new_labels[kept] = merge_mapping[sorting.labels[kept]]
|
|
67
|
+
new_labels = sorting.labels.copy()
|
|
68
|
+
apply_label_remapping_in_place(new_labels, merge_mapping)
|
|
65
69
|
|
|
66
70
|
if shifts is None:
|
|
67
71
|
return sorting.ephemeral_replace(labels=new_labels)
|
|
@@ -99,7 +103,6 @@ def hierarchical_cluster(
|
|
|
99
103
|
threshold=1.0,
|
|
100
104
|
eps=1e-5,
|
|
101
105
|
):
|
|
102
|
-
""""""
|
|
103
106
|
n = distances.shape[0]
|
|
104
107
|
assert eps < threshold # that would be confusing.
|
|
105
108
|
if n <= 1:
|
|
@@ -117,8 +120,8 @@ def hierarchical_cluster(
|
|
|
117
120
|
if labels is None:
|
|
118
121
|
return None, np.arange(n)
|
|
119
122
|
else:
|
|
120
|
-
ids =
|
|
121
|
-
return labels, ids
|
|
123
|
+
ids, _, _ = pos_int_unique_and_counts(labels)
|
|
124
|
+
return labels, ids
|
|
122
125
|
|
|
123
126
|
if not finite.all():
|
|
124
127
|
inf = max(0, pdist[finite].max()) + threshold + 1.0
|
|
@@ -175,8 +178,7 @@ def sparsify_labels(
|
|
|
175
178
|
) -> dict[int, np.ndarray]:
|
|
176
179
|
assert labels.ndim == 1
|
|
177
180
|
if ids is None:
|
|
178
|
-
ids =
|
|
179
|
-
ids = ids[ids >= 0]
|
|
181
|
+
ids, _, _ = pos_int_unique_and_counts(labels)
|
|
180
182
|
inj = {}
|
|
181
183
|
for j in ids:
|
|
182
184
|
inj[j] = np.flatnonzero(labels == j)
|
|
@@ -265,7 +267,7 @@ def maximal_leaf_groups(
|
|
|
265
267
|
|
|
266
268
|
assert max(map(len, groups)) <= max_group_size
|
|
267
269
|
assert sum(map(len, groups)) == n
|
|
268
|
-
assert
|
|
270
|
+
assert {gv for g in groups for gv in g} == set(range(n))
|
|
269
271
|
|
|
270
272
|
groups = [tuple(sorted(g)) for g in groups]
|
|
271
273
|
|
|
@@ -329,6 +331,7 @@ def reorder_by_depth(
|
|
|
329
331
|
spatial_footprints: np.ndarray | None = None,
|
|
330
332
|
geom: np.ndarray | None = None,
|
|
331
333
|
centroids: np.ndarray | None = None,
|
|
334
|
+
in_place: bool = False,
|
|
332
335
|
) -> tuple[DARTsortSorting, np.ndarray]:
|
|
333
336
|
"""Reorder cluster labels so that centroid depth is increasing
|
|
334
337
|
|
|
@@ -346,11 +349,8 @@ def reorder_by_depth(
|
|
|
346
349
|
reorder: np.ndarray
|
|
347
350
|
reorder[j] is the new label of original unit j.
|
|
348
351
|
"""
|
|
352
|
+
sorting = sorting.flatten(include_gmm_properties=True, in_place=in_place)
|
|
349
353
|
assert sorting.labels is not None
|
|
350
|
-
kept = np.flatnonzero(sorting.labels >= 0)
|
|
351
|
-
kept_labels = sorting.labels[kept]
|
|
352
|
-
|
|
353
|
-
units, kept_labels = np.unique(kept_labels, return_inverse=True)
|
|
354
354
|
|
|
355
355
|
if geom is None and motion is not None:
|
|
356
356
|
geom = motion.rgeom
|
|
@@ -359,26 +359,19 @@ def reorder_by_depth(
|
|
|
359
359
|
assert centroids is None
|
|
360
360
|
assert geom is not None
|
|
361
361
|
assert spatial_footprints.shape[1] == geom.shape[0]
|
|
362
|
-
assert spatial_footprints.shape[0] == units.shape[0]
|
|
363
362
|
w = spatial_footprints / spatial_footprints.sum(1, keepdims=True)
|
|
364
363
|
assert np.isfinite(w).all()
|
|
365
364
|
centroids = w @ geom[:, 1]
|
|
366
365
|
|
|
367
366
|
if centroids is None:
|
|
368
|
-
|
|
367
|
+
z = sorting.load_dataset("point_source_localizations", sl=(slice(None), 2))
|
|
369
368
|
if motion is not None:
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
centroids = np.zeros(units.size)
|
|
373
|
-
for u in range(units.size):
|
|
374
|
-
inu = np.flatnonzero(kept_labels == u)
|
|
375
|
-
centroids[u] = np.median(depths[inu])
|
|
376
|
-
assert centroids.shape[0] == units.shape[0]
|
|
369
|
+
z = motion.correct_s(sorting.times_seconds, z)
|
|
370
|
+
centroids = mean_by_label_1d(sorting, z)
|
|
377
371
|
|
|
378
|
-
labels = sorting.labels.copy()
|
|
379
|
-
# this one is some food for thought, lol.
|
|
372
|
+
labels = sorting.labels if in_place else sorting.labels.copy()
|
|
380
373
|
reorder = np.argsort(np.argsort(centroids, kind="stable"), kind="stable")
|
|
381
|
-
labels
|
|
374
|
+
apply_label_remapping_in_place(labels, reorder)
|
|
382
375
|
reordered_sorting = sorting.ephemeral_replace(labels=labels)
|
|
383
376
|
|
|
384
377
|
return reordered_sorting, reorder
|
|
@@ -527,7 +520,7 @@ def get_main_channel_pcs(
|
|
|
527
520
|
return features
|
|
528
521
|
|
|
529
522
|
|
|
530
|
-
def
|
|
523
|
+
def decrumb_labels(labels: np.ndarray, min_size: int = 5, in_place=False, flatten=True):
|
|
531
524
|
"""Remove small units
|
|
532
525
|
|
|
533
526
|
Parameters
|
|
@@ -543,27 +536,43 @@ def decrumb(labels: np.ndarray, min_size: int = 5, in_place=False, flatten=True)
|
|
|
543
536
|
labels
|
|
544
537
|
The (flattened) decrumbed labels.
|
|
545
538
|
"""
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
labels = labels if in_place else labels.copy()
|
|
549
|
-
|
|
550
|
-
units_sparse, counts_sparse = np.unique(labels_kept, return_counts=True)
|
|
551
|
-
if not units_sparse.size:
|
|
539
|
+
units, counts, _ = pos_int_unique_and_counts(labels)
|
|
540
|
+
if not units.size:
|
|
552
541
|
return labels
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
big_enough = counts >= min_size
|
|
560
|
-
k1 = big_enough.sum()
|
|
561
|
-
units[np.logical_not(big_enough)] = -1
|
|
542
|
+
all_big = counts.min() >= min_size
|
|
543
|
+
flat_ok = (not flatten) or np.array_equal(units, np.arange(len(units)))
|
|
544
|
+
if all_big and flat_ok:
|
|
545
|
+
return labels
|
|
546
|
+
remapping = np.full((units.max() + 1,), -1, dtype=labels.dtype)
|
|
547
|
+
kept_units = units[counts >= min_size]
|
|
562
548
|
if flatten:
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
549
|
+
remapping[kept_units] = np.arange(len(kept_units))
|
|
550
|
+
else:
|
|
551
|
+
remapping[kept_units] = kept_units
|
|
552
|
+
new_labels = labels if in_place else labels.copy()
|
|
553
|
+
apply_label_remapping_in_place(new_labels, remapping)
|
|
554
|
+
logger.dartsortdebug(f"decrumb ({min_size}): {units.size}->{kept_units.size}.")
|
|
555
|
+
return new_labels
|
|
566
556
|
|
|
567
|
-
labels[kept] = units[labels_kept]
|
|
568
557
|
|
|
569
|
-
|
|
558
|
+
def decrumb(
|
|
559
|
+
sorting: DARTsortSorting, min_size: int = 5, in_place=False, flatten=True
|
|
560
|
+
) -> DARTsortSorting:
|
|
561
|
+
assert sorting.labels is not None
|
|
562
|
+
units, counts, _ = pos_int_unique_and_counts(sorting.labels)
|
|
563
|
+
if not units.size:
|
|
564
|
+
return sorting
|
|
565
|
+
all_big = counts.min() >= min_size
|
|
566
|
+
flat_ok = (not flatten) or np.array_equal(units, np.arange(len(units)))
|
|
567
|
+
if all_big and flat_ok:
|
|
568
|
+
return sorting
|
|
569
|
+
|
|
570
|
+
remapping = np.full((units.max() + 1,), -1, dtype=sorting.labels.dtype)
|
|
571
|
+
kept_units = units[counts >= min_size]
|
|
572
|
+
remapping[kept_units] = kept_units
|
|
573
|
+
new_labels = sorting.labels if in_place else sorting.labels.copy()
|
|
574
|
+
apply_label_remapping_in_place(new_labels, remapping)
|
|
575
|
+
sorting = sorting.ephemeral_replace(labels=new_labels)
|
|
576
|
+
if flatten:
|
|
577
|
+
sorting = sorting.flatten(in_place=in_place)
|
|
578
|
+
return sorting
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import gc
|
|
2
|
-
from
|
|
2
|
+
from collections.abc import Sequence
|
|
3
|
+
from typing import TYPE_CHECKING, Literal, Self, cast
|
|
3
4
|
|
|
4
5
|
import numpy as np
|
|
5
6
|
import sklearn.cluster
|
|
@@ -473,7 +474,7 @@ class DensityPeaksClusterer(Clusterer):
|
|
|
473
474
|
del kres
|
|
474
475
|
cleanup_and_log_gpu_usage(self.computation_cfg, "DPC->kmeans")
|
|
475
476
|
|
|
476
|
-
labels = cluster_util.
|
|
477
|
+
labels = cluster_util.decrumb_labels(
|
|
477
478
|
labels, min_size=self.remove_clusters_smaller_than, in_place=True
|
|
478
479
|
)
|
|
479
480
|
|