DARTsort 0.5.27__tar.gz → 0.5.28__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.27 → dartsort-0.5.28}/PKG-INFO +1 -1
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/config.py +7 -2
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/peel_lib.py +58 -27
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/shaver.py +2 -2
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/subtract.py +42 -45
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/subtract_util.py +127 -52
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/threshold.py +146 -18
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/decollider.py +130 -66
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/matched_filter_net.py +1 -2
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/pipeline.py +19 -1
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/temporal_pca.py +23 -17
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/transform_base.py +5 -25
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/internal_config.py +32 -6
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/preprocess_util.py +26 -17
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/analysis_plots.py +13 -5
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/conftest.py +3 -3
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_subtract.py +106 -15
- dartsort-0.5.28/tests/test_threshold.py +80 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_transform.py +13 -2
- dartsort-0.5.27/tests/test_threshold.py +0 -9
- {dartsort-0.5.27 → dartsort-0.5.28}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/.gitignore +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/README.md +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/docs/index.md +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/docs/main_api.md +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/environment.yml +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/notebook/.keep +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/notebook/fit_and_batch_matching.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/notebook/ibl_dandi_example.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/pyproject.toml +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/setup.cfg +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/__main__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/agglomerate.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/cluster_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/clustering.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/clustering_features.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/kmeans.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/mixture.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/clustering/refine_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/analysis.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/comparison.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/hybridkit.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/simkit.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/main.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/peel_base.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/_multichan_denoiser_kit.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/amplitudes.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/mixture_classifier.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/vq_matched_filter.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/data_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/main_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/motion.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/noise_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/peel_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/spiketorch.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/mixture.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/vismain.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/SOURCES.txt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_agglomerate.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_alignment.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_clustering.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_clustering_features.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_config.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_data_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_density.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_detect.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_hybridkit.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_linalg.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_matching.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_modeling.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_simkit.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_templates.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.27 → dartsort-0.5.28}/zensical.toml +0 -0
|
@@ -4,6 +4,7 @@ from typing import Annotated, Literal
|
|
|
4
4
|
from pydantic import Field
|
|
5
5
|
|
|
6
6
|
from .util.internal_config import (
|
|
7
|
+
DetectionProposal,
|
|
7
8
|
InterpKernel,
|
|
8
9
|
InterpMethod,
|
|
9
10
|
KmeansppSelection,
|
|
@@ -12,6 +13,7 @@ from .util.internal_config import (
|
|
|
12
13
|
PreprocessingStrategy,
|
|
13
14
|
RealignStrategy,
|
|
14
15
|
TemplateSVDMethod,
|
|
16
|
+
TPCADenoiserFitStage,
|
|
15
17
|
WhiteningEstimator,
|
|
16
18
|
WhiteningStrategy,
|
|
17
19
|
default_pretrained_path, # noqa
|
|
@@ -277,10 +279,13 @@ class DeveloperConfig(DARTsortUserConfig):
|
|
|
277
279
|
nn_denoiser_noise_waveforms: int = 100 * 256
|
|
278
280
|
nn_denoiser_extra_kwargs: dict | None = None
|
|
279
281
|
score_filter_radius_um: float = 0.0
|
|
280
|
-
detection_proposal:
|
|
282
|
+
detection_proposal: DetectionProposal = "voltage"
|
|
281
283
|
proposal_filters: int = 1
|
|
282
|
-
|
|
284
|
+
propose_with_score_net: bool = False
|
|
285
|
+
proposal_threshold: float = 5.0
|
|
286
|
+
score_net_threshold: float = 5.0
|
|
283
287
|
do_tpca_denoise: bool = True
|
|
288
|
+
tpca_denoise_fit: TPCADenoiserFitStage | None = None
|
|
284
289
|
first_denoiser_thinning: float = 0.0
|
|
285
290
|
first_denoiser_spatial_dedup_radius: float = 100.0
|
|
286
291
|
realign_to_denoiser: bool = True
|
|
@@ -12,6 +12,7 @@ from torch import Tensor
|
|
|
12
12
|
from ..detect import detect_and_deduplicate
|
|
13
13
|
from ..util.internal_config import (
|
|
14
14
|
ComputationConfig,
|
|
15
|
+
FeaturizationConfig,
|
|
15
16
|
FitSamplingConfig,
|
|
16
17
|
PeakSign,
|
|
17
18
|
ThresholdingConfig,
|
|
@@ -20,7 +21,6 @@ from ..util.internal_config import (
|
|
|
20
21
|
from ..util.job_util import ensure_computation_config
|
|
21
22
|
from ..util.spiketorch import grab_spikes, subtract_spikes_
|
|
22
23
|
from ..util.torch_util import torch_compile
|
|
23
|
-
from ..util.waveform_util import make_channel_index
|
|
24
24
|
from .peel_base import PeelingBatchResult
|
|
25
25
|
|
|
26
26
|
if TYPE_CHECKING:
|
|
@@ -120,7 +120,7 @@ def flatten_denan_and_whiten_batched(
|
|
|
120
120
|
def threshold_chunk(
|
|
121
121
|
traces,
|
|
122
122
|
channel_index,
|
|
123
|
-
|
|
123
|
+
voltage_threshold=4.0,
|
|
124
124
|
peak_sign: PeakSign = "both",
|
|
125
125
|
peak_channel_index=None,
|
|
126
126
|
dedup_channel_index=None,
|
|
@@ -140,10 +140,9 @@ def threshold_chunk(
|
|
|
140
140
|
rg=None,
|
|
141
141
|
quiet=False,
|
|
142
142
|
) -> PeelingBatchResult:
|
|
143
|
-
n_index = channel_index.shape[1]
|
|
144
143
|
times_rel, channels, energies = detect_and_deduplicate(
|
|
145
144
|
traces,
|
|
146
|
-
threshold=
|
|
145
|
+
threshold=voltage_threshold,
|
|
147
146
|
peak_channel_index=peak_channel_index,
|
|
148
147
|
dedup_neighborhoods=dedup_channel_index,
|
|
149
148
|
peak_sign=peak_sign,
|
|
@@ -153,6 +152,46 @@ def threshold_chunk(
|
|
|
153
152
|
return_energies=True,
|
|
154
153
|
trough_priority=trough_priority,
|
|
155
154
|
)
|
|
155
|
+
return extract_detections(
|
|
156
|
+
traces,
|
|
157
|
+
times_rel,
|
|
158
|
+
channels,
|
|
159
|
+
energies,
|
|
160
|
+
channel_index=channel_index,
|
|
161
|
+
trough_offset_samples=trough_offset_samples,
|
|
162
|
+
spike_length_samples=spike_length_samples,
|
|
163
|
+
left_margin=left_margin,
|
|
164
|
+
right_margin=right_margin,
|
|
165
|
+
max_spikes_per_chunk=max_spikes_per_chunk,
|
|
166
|
+
thinning=thinning,
|
|
167
|
+
time_jitter=time_jitter,
|
|
168
|
+
spatial_jitter_channel_index=spatial_jitter_channel_index,
|
|
169
|
+
return_waveforms=return_waveforms,
|
|
170
|
+
rg=rg,
|
|
171
|
+
quiet=quiet,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def extract_detections(
|
|
176
|
+
traces,
|
|
177
|
+
times_rel,
|
|
178
|
+
channels,
|
|
179
|
+
energies,
|
|
180
|
+
*,
|
|
181
|
+
channel_index,
|
|
182
|
+
trough_offset_samples=42,
|
|
183
|
+
spike_length_samples=121,
|
|
184
|
+
left_margin=0,
|
|
185
|
+
right_margin=0,
|
|
186
|
+
max_spikes_per_chunk=None,
|
|
187
|
+
thinning=0.0,
|
|
188
|
+
time_jitter=0,
|
|
189
|
+
spatial_jitter_channel_index=None,
|
|
190
|
+
return_waveforms=True,
|
|
191
|
+
rg=None,
|
|
192
|
+
quiet=False,
|
|
193
|
+
) -> PeelingBatchResult:
|
|
194
|
+
n_index = channel_index.shape[1]
|
|
156
195
|
if not times_rel.numel():
|
|
157
196
|
return PeelingBatchResult(
|
|
158
197
|
n_spikes=0,
|
|
@@ -281,7 +320,7 @@ def perturb_detections(
|
|
|
281
320
|
n = len(times_rel)
|
|
282
321
|
if time_jitter:
|
|
283
322
|
assert rg is not None
|
|
284
|
-
jitter = rg.integers(low=-time_jitter, high=time_jitter + 1)
|
|
323
|
+
jitter = rg.integers(low=-time_jitter, high=time_jitter + 1, size=n)
|
|
285
324
|
times_rel = times_rel + torch.asarray(
|
|
286
325
|
jitter, dtype=times_rel.dtype, device=times_rel.device
|
|
287
326
|
)
|
|
@@ -436,8 +475,8 @@ def threshold_to_fit(
|
|
|
436
475
|
recording: BaseRecording,
|
|
437
476
|
waveform_cfg: WaveformConfig,
|
|
438
477
|
channel_index: Tensor,
|
|
439
|
-
spatial_dedup_radius: float | None,
|
|
440
478
|
threshold_cfg: ThresholdingConfig,
|
|
479
|
+
featurization_cfg: FeaturizationConfig,
|
|
441
480
|
sampling_cfg: FitSamplingConfig,
|
|
442
481
|
max_waveforms_fit: int | None = None,
|
|
443
482
|
n_residual_snips: int | None = None,
|
|
@@ -448,34 +487,23 @@ def threshold_to_fit(
|
|
|
448
487
|
|
|
449
488
|
Used by subtraction to fit initial NN denoisers.
|
|
450
489
|
"""
|
|
451
|
-
from ..transform import Waveform, WaveformPipeline
|
|
452
490
|
from ..util.data_util import subsample_waveforms
|
|
453
491
|
from .threshold import Threshold
|
|
454
492
|
|
|
455
493
|
computation_cfg = ensure_computation_config(computation_cfg)
|
|
456
494
|
|
|
457
|
-
|
|
458
|
-
waveform_node = Waveform(
|
|
459
|
-
channel_index=channel_index,
|
|
460
|
-
waveform_cfg=waveform_cfg,
|
|
461
|
-
sampling_frequency=recording.sampling_frequency,
|
|
462
|
-
)
|
|
463
|
-
waveform_pipeline = WaveformPipeline([waveform_node])
|
|
464
|
-
|
|
465
|
-
if spatial_dedup_radius:
|
|
466
|
-
dn_dedup_ci = make_channel_index(geom, spatial_dedup_radius, to_torch=True)
|
|
467
|
-
dn_dedup_ci = dn_dedup_ci.to(channel_index)
|
|
468
|
-
else:
|
|
469
|
-
dn_dedup_ci = channel_index
|
|
470
|
-
trainer = Threshold(
|
|
495
|
+
trainer = Threshold.from_config(
|
|
471
496
|
recording=recording,
|
|
472
|
-
channel_index=channel_index,
|
|
473
|
-
featurization_pipeline=waveform_pipeline,
|
|
474
|
-
p=threshold_cfg,
|
|
475
497
|
waveform_cfg=waveform_cfg,
|
|
476
|
-
|
|
477
|
-
|
|
498
|
+
thresholding_cfg=threshold_cfg,
|
|
499
|
+
featurization_cfg=featurization_cfg,
|
|
500
|
+
sampling_cfg=sampling_cfg,
|
|
501
|
+
extract_channel_index=channel_index,
|
|
478
502
|
)
|
|
503
|
+
assert trainer.featurization_pipeline is not None
|
|
504
|
+
assert "waveforms" in {
|
|
505
|
+
ds.name for ds in trainer.featurization_pipeline.spike_datasets()
|
|
506
|
+
}
|
|
479
507
|
|
|
480
508
|
if max_waveforms_fit is None:
|
|
481
509
|
max_waveforms_fit = sampling_cfg.max_waveforms_fit
|
|
@@ -490,6 +518,9 @@ def threshold_to_fit(
|
|
|
490
518
|
with TemporaryDirectory(dir=tmp_dir) as temp_dir:
|
|
491
519
|
temp_hdf5_filename = Path(temp_dir) / "subtraction_denoiser0_fit.h5"
|
|
492
520
|
try:
|
|
521
|
+
trainer.load_or_fit_and_save_models(
|
|
522
|
+
Path(temp_dir) / "threshold_models", computation_cfg=computation_cfg
|
|
523
|
+
)
|
|
493
524
|
trainer.run_subsampled_peeling(
|
|
494
525
|
temp_hdf5_filename,
|
|
495
526
|
stop_after_n_waveforms=max_waveforms_fit,
|
|
@@ -515,7 +546,7 @@ def threshold_to_fit(
|
|
|
515
546
|
raise ValueError(
|
|
516
547
|
"Found no spikes when thresholding to get model fitting data. "
|
|
517
548
|
"This usually indicates a preprocessing issue, since it means "
|
|
518
|
-
f"that no spikes could be found at threshold {threshold_cfg.
|
|
549
|
+
f"that no spikes could be found at threshold {threshold_cfg.voltage_threshold}."
|
|
519
550
|
)
|
|
520
551
|
|
|
521
552
|
# fit the thing
|
|
@@ -59,7 +59,7 @@ class Shaver(BasePeeler):
|
|
|
59
59
|
fixed_property_keys=fixed_prop_keys,
|
|
60
60
|
)
|
|
61
61
|
self.p = p
|
|
62
|
-
self.peel_kind = f"Shave {p.
|
|
62
|
+
self.peel_kind = f"Shave {p.voltage_threshold}"
|
|
63
63
|
self.denoising_pipeline = denoising_pipeline
|
|
64
64
|
self.dedup_batch_size = self.nearest_batch_length()
|
|
65
65
|
|
|
@@ -135,7 +135,7 @@ class Shaver(BasePeeler):
|
|
|
135
135
|
channel_index=self.b.channel_index,
|
|
136
136
|
denoising_pipeline=self.denoising_pipeline,
|
|
137
137
|
residnorm_decrease_threshold=self.p.shave_score,
|
|
138
|
-
detection_threshold=self.p.
|
|
138
|
+
detection_threshold=self.p.voltage_threshold,
|
|
139
139
|
peak_sign=self.p.peak_sign,
|
|
140
140
|
dedup_channel_index=self.b.dedup_channel_index,
|
|
141
141
|
trough_offset_samples=self.trough_offset_samples,
|
|
@@ -12,9 +12,7 @@ import torch
|
|
|
12
12
|
from spikeinterface.core import BaseRecording
|
|
13
13
|
|
|
14
14
|
from ..transform import (
|
|
15
|
-
BaseTemporalPCA,
|
|
16
15
|
Voltage,
|
|
17
|
-
VQMatchedFilter,
|
|
18
16
|
Waveform,
|
|
19
17
|
WaveformPipeline,
|
|
20
18
|
WaveformWhitener,
|
|
@@ -41,7 +39,13 @@ from ..util.waveform_util import (
|
|
|
41
39
|
)
|
|
42
40
|
from .peel_base import BasePeeler, PeelingBatchResult
|
|
43
41
|
from .peel_lib import threshold_to_fit
|
|
44
|
-
from .subtract_util import
|
|
42
|
+
from .subtract_util import (
|
|
43
|
+
ChunkSubtracter,
|
|
44
|
+
make_peak_proposer,
|
|
45
|
+
proposal_filters_from_pipeline,
|
|
46
|
+
proposal_fit_featurization_cfg,
|
|
47
|
+
with_proposal_filters,
|
|
48
|
+
)
|
|
45
49
|
|
|
46
50
|
|
|
47
51
|
class SubtractionPeeler(BasePeeler):
|
|
@@ -60,6 +64,7 @@ class SubtractionPeeler(BasePeeler):
|
|
|
60
64
|
p: SubtractionConfig = default_subtraction_cfg,
|
|
61
65
|
waveform_cfg: WaveformConfig = default_waveform_cfg,
|
|
62
66
|
fit_sampling_cfg: FitSamplingConfig = default_peeling_fit_sampling_cfg,
|
|
67
|
+
featurization_cfg: FeaturizationConfig | None = None,
|
|
63
68
|
save_iteration=False,
|
|
64
69
|
save_residnorm_decrease=False,
|
|
65
70
|
save_collidedness=False,
|
|
@@ -89,6 +94,8 @@ class SubtractionPeeler(BasePeeler):
|
|
|
89
94
|
self.save_iteration = save_iteration
|
|
90
95
|
self.save_residnorm_decrease = save_residnorm_decrease
|
|
91
96
|
self.save_collidedness = save_collidedness
|
|
97
|
+
# non-voltage proposal may need this config
|
|
98
|
+
self.featurization_cfg = featurization_cfg
|
|
92
99
|
self.dedup_batch_size = self.nearest_batch_length()
|
|
93
100
|
if self.p.whiten:
|
|
94
101
|
self.threshold = self.p.threshold_before_whitening
|
|
@@ -181,8 +188,13 @@ class SubtractionPeeler(BasePeeler):
|
|
|
181
188
|
return self._subtracters.subtracter
|
|
182
189
|
|
|
183
190
|
def _build_chunk_subtracter(self) -> ChunkSubtracter:
|
|
191
|
+
proposal = self.p.detection_proposal
|
|
192
|
+
if self.b.proposal_filters is None:
|
|
193
|
+
# hit during the fit pass before we have that gizmo
|
|
194
|
+
proposal = "voltage"
|
|
184
195
|
proposer = make_peak_proposer(
|
|
185
196
|
self.p,
|
|
197
|
+
proposal,
|
|
186
198
|
spike_length_samples=self.spike_length_samples,
|
|
187
199
|
trough_offset_samples=self.trough_offset_samples,
|
|
188
200
|
peak_channel_index=self.b.peak_channel_index,
|
|
@@ -269,32 +281,18 @@ class SubtractionPeeler(BasePeeler):
|
|
|
269
281
|
def _install_proposal_filters(self):
|
|
270
282
|
assert self.featurization_pipeline is not None
|
|
271
283
|
assert not self.featurization_pipeline.needs_fit()
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
else:
|
|
279
|
-
nodes = [
|
|
280
|
-
t
|
|
281
|
-
for t in self.featurization_pipeline.transformers
|
|
282
|
-
if type(t) is BaseTemporalPCA
|
|
283
|
-
]
|
|
284
|
-
if len(nodes) != 1:
|
|
285
|
-
panic(
|
|
286
|
-
f"detection_proposal={self.p.detection_proposal!r} needs exactly one "
|
|
287
|
-
f"proposal model in the featurization pipeline, found {len(nodes)}."
|
|
288
|
-
)
|
|
289
|
-
node = nodes[0]
|
|
290
|
-
filters = node.b.components[: self.p.proposal_filters]
|
|
291
|
-
filters = filters.to(self.b.sub_channel_index.device).clone()
|
|
284
|
+
filters, trough_offset = proposal_filters_from_pipeline(
|
|
285
|
+
self.featurization_pipeline,
|
|
286
|
+
self.p.detection_proposal,
|
|
287
|
+
self.p.proposal_filters,
|
|
288
|
+
)
|
|
289
|
+
filters = filters.to(self.b.sub_channel_index.device)
|
|
292
290
|
if self.b.proposal_filters is None:
|
|
293
291
|
self.del_none_buffer("proposal_filters")
|
|
294
292
|
self.register_buffer("proposal_filters", filters)
|
|
295
293
|
else:
|
|
296
294
|
self.proposal_filters = filters
|
|
297
|
-
self.proposal_filter_trough_offset =
|
|
295
|
+
self.proposal_filter_trough_offset = trough_offset
|
|
298
296
|
|
|
299
297
|
def save_models(self, save_folder: str | Path):
|
|
300
298
|
sub_denoise_pt = Path(save_folder) / "subtraction_denoising_pipeline.pt"
|
|
@@ -337,14 +335,10 @@ class SubtractionPeeler(BasePeeler):
|
|
|
337
335
|
)
|
|
338
336
|
|
|
339
337
|
# handle peak proposal models
|
|
340
|
-
proposal = subtraction_cfg.detection_proposal
|
|
341
338
|
denoising_cfg = subtraction_cfg.subtraction_denoising_cfg
|
|
342
|
-
if
|
|
339
|
+
if subtraction_cfg.propose_with_score_net:
|
|
343
340
|
if not denoising_cfg.score_filter_radius_um:
|
|
344
|
-
panic(
|
|
345
|
-
"detection_proposal='score_net' needs score_filter_radius_um "
|
|
346
|
-
"set in the subtraction denoising config."
|
|
347
|
-
)
|
|
341
|
+
panic()
|
|
348
342
|
else:
|
|
349
343
|
subtraction_cfg = replace(
|
|
350
344
|
subtraction_cfg,
|
|
@@ -352,19 +346,11 @@ class SubtractionPeeler(BasePeeler):
|
|
|
352
346
|
denoising_cfg, score_filter_radius_um=None
|
|
353
347
|
),
|
|
354
348
|
)
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
)
|
|
361
|
-
featurization_cfg = replace(
|
|
362
|
-
featurization_cfg, learn_cleaned_tpca_basis=True
|
|
363
|
-
)
|
|
364
|
-
elif proposal == "vq":
|
|
365
|
-
featurization_cfg = replace(
|
|
366
|
-
featurization_cfg, vq_proposal_filters=subtraction_cfg.proposal_filters
|
|
367
|
-
)
|
|
349
|
+
featurization_cfg = with_proposal_filters(
|
|
350
|
+
featurization_cfg,
|
|
351
|
+
subtraction_cfg.detection_proposal,
|
|
352
|
+
subtraction_cfg.proposal_filters,
|
|
353
|
+
)
|
|
368
354
|
|
|
369
355
|
# construct denoising and featurization pipelines
|
|
370
356
|
subtraction_denoising_pipeline = WaveformPipeline.from_config(
|
|
@@ -402,6 +388,7 @@ class SubtractionPeeler(BasePeeler):
|
|
|
402
388
|
p=subtraction_cfg,
|
|
403
389
|
waveform_cfg=waveform_cfg,
|
|
404
390
|
fit_sampling_cfg=sampling_cfg,
|
|
391
|
+
featurization_cfg=featurization_cfg,
|
|
405
392
|
save_iteration=subtraction_cfg.save_iteration,
|
|
406
393
|
save_residnorm_decrease=subtraction_cfg.save_residnorm_decrease,
|
|
407
394
|
save_collidedness=save_collidedness,
|
|
@@ -641,7 +628,13 @@ class SubtractionPeeler(BasePeeler):
|
|
|
641
628
|
|
|
642
629
|
def _threshold_to_fit(self, tmp_dir, fit_pipeline, computation_cfg):
|
|
643
630
|
threshold_cfg = ThresholdingConfig(
|
|
644
|
-
|
|
631
|
+
detection_proposal=self.p.detection_proposal,
|
|
632
|
+
proposal_filters=self.p.proposal_filters,
|
|
633
|
+
voltage_threshold=self.p.voltage_threshold,
|
|
634
|
+
proposal_threshold=self.p.proposal_threshold,
|
|
635
|
+
spatial_dedup_radius_um=(
|
|
636
|
+
self.p.first_denoiser_spatial_dedup_radius or self.p.subtract_radius_um
|
|
637
|
+
),
|
|
645
638
|
peak_sign=self.p.peak_sign,
|
|
646
639
|
relative_peak_radius_um=self.p.relative_peak_radius_um,
|
|
647
640
|
relative_peak_radius_samples=self.p.relative_peak_radius_samples,
|
|
@@ -665,8 +658,12 @@ class SubtractionPeeler(BasePeeler):
|
|
|
665
658
|
recording=self.recording,
|
|
666
659
|
waveform_cfg=self.waveform_cfg,
|
|
667
660
|
channel_index=self.b.sub_channel_index,
|
|
668
|
-
spatial_dedup_radius=self.p.first_denoiser_spatial_dedup_radius,
|
|
669
661
|
threshold_cfg=threshold_cfg,
|
|
662
|
+
featurization_cfg=proposal_fit_featurization_cfg(
|
|
663
|
+
self.featurization_cfg,
|
|
664
|
+
self.p.detection_proposal,
|
|
665
|
+
self.p.proposal_filters,
|
|
666
|
+
),
|
|
670
667
|
sampling_cfg=sampling_cfg,
|
|
671
668
|
max_waveforms_fit=self.p.first_denoiser_max_waveforms_fit,
|
|
672
669
|
n_residual_snips=self.p.first_denoiser_noise_snips,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from dataclasses import replace
|
|
1
2
|
from typing import TYPE_CHECKING, Literal, Protocol, Self
|
|
2
3
|
|
|
3
4
|
import torch
|
|
@@ -11,7 +12,13 @@ from ..detect.detect import (
|
|
|
11
12
|
peak_sign_to_pos,
|
|
12
13
|
update_peak_map,
|
|
13
14
|
)
|
|
14
|
-
from ..
|
|
15
|
+
from ..transform import BaseTemporalPCA, VQMatchedFilter
|
|
16
|
+
from ..util.internal_config import (
|
|
17
|
+
DetectionProposal,
|
|
18
|
+
FeaturizationConfig,
|
|
19
|
+
PeakSign,
|
|
20
|
+
SubtractionConfig,
|
|
21
|
+
)
|
|
15
22
|
from ..util.py_util import databag, panic
|
|
16
23
|
from ..util.spiketorch import grab_spikes, subtract_spikes_
|
|
17
24
|
from ..util.torch_util import BModule
|
|
@@ -165,7 +172,9 @@ class GlobalPeakProposer:
|
|
|
165
172
|
reference = self._full_peak_map(residual)
|
|
166
173
|
n_bad = int((reference != self.peak_map).sum())
|
|
167
174
|
if n_bad:
|
|
168
|
-
|
|
175
|
+
raise ValueError(
|
|
176
|
+
f"patched peak map differs from full recompute at {n_bad} sites"
|
|
177
|
+
)
|
|
169
178
|
|
|
170
179
|
def propose_peaks(
|
|
171
180
|
self, residual: Tensor, detection_mask: Tensor | None
|
|
@@ -407,10 +416,14 @@ class MatchedFilterProposer:
|
|
|
407
416
|
bad = int(off.sum())
|
|
408
417
|
if bad:
|
|
409
418
|
worst = (reference - self.field).abs().nan_to_num().max()
|
|
410
|
-
|
|
419
|
+
raise ValueError(
|
|
420
|
+
f"patched score field differs at {bad} sites, worst {worst:.3e}"
|
|
421
|
+
)
|
|
411
422
|
bad = int((self._full_peak_map() != self.peak_map).sum())
|
|
412
423
|
if bad:
|
|
413
|
-
|
|
424
|
+
raise ValueError(
|
|
425
|
+
f"patched score peak map differs from full recompute at {bad} sites"
|
|
426
|
+
)
|
|
414
427
|
|
|
415
428
|
|
|
416
429
|
@databag
|
|
@@ -455,7 +468,7 @@ class LinearMatchedFilterProposer:
|
|
|
455
468
|
return self.spike_length_samples + nbefore + nafter
|
|
456
469
|
|
|
457
470
|
def setup(self, residual: Tensor) -> None:
|
|
458
|
-
self.field = self.
|
|
471
|
+
self.field = self.full_proposal_score_field(residual)
|
|
459
472
|
self.peak_map = self._full_peak_map()
|
|
460
473
|
|
|
461
474
|
def cleanup(self) -> None:
|
|
@@ -522,6 +535,12 @@ class LinearMatchedFilterProposer:
|
|
|
522
535
|
out[i0:i1] = self.score_from_responses(responses).T
|
|
523
536
|
return out
|
|
524
537
|
|
|
538
|
+
def full_proposal_score_field(self, residual: Tensor) -> Tensor:
|
|
539
|
+
field = residual.new_full(residual.shape, -torch.inf)
|
|
540
|
+
dense = self.dense(residual[:, :-1])
|
|
541
|
+
field[self.trough_offset : self.trough_offset + dense.shape[0], :-1] = dense
|
|
542
|
+
return field
|
|
543
|
+
|
|
525
544
|
# -- the patch
|
|
526
545
|
|
|
527
546
|
def _patch_field(self, residual: Tensor, peaks: AcceptedPeaks) -> Tensor:
|
|
@@ -557,12 +576,6 @@ class LinearMatchedFilterProposer:
|
|
|
557
576
|
|
|
558
577
|
# -- for audit
|
|
559
578
|
|
|
560
|
-
def _full_field(self, residual: Tensor) -> Tensor:
|
|
561
|
-
field = residual.new_full(residual.shape, -torch.inf)
|
|
562
|
-
dense = self.dense(residual[:, :-1])
|
|
563
|
-
field[self.trough_offset : self.trough_offset + dense.shape[0], :-1] = dense
|
|
564
|
-
return field
|
|
565
|
-
|
|
566
579
|
def _full_peak_map(self) -> Tensor:
|
|
567
580
|
assert self.field is not None
|
|
568
581
|
return is_extreme_transpose_no_pad(
|
|
@@ -573,17 +586,21 @@ class LinearMatchedFilterProposer:
|
|
|
573
586
|
|
|
574
587
|
def _check(self, residual: Tensor) -> None:
|
|
575
588
|
assert self.field is not None and self.peak_map is not None
|
|
576
|
-
reference = self.
|
|
589
|
+
reference = self.full_proposal_score_field(residual)
|
|
577
590
|
off = reference.isfinite().logical_and_(
|
|
578
591
|
torch.isclose(reference, self.field, rtol=1e-4, atol=1e-5).logical_not_()
|
|
579
592
|
)
|
|
580
593
|
bad = int(off.sum())
|
|
581
594
|
if bad:
|
|
582
595
|
worst = (reference - self.field).abs().nan_to_num().max()
|
|
583
|
-
|
|
596
|
+
raise ValueError(
|
|
597
|
+
f"patched linear field differs at {bad} sites, worst {worst:.3e}"
|
|
598
|
+
)
|
|
584
599
|
bad = int((self._full_peak_map() != self.peak_map).sum())
|
|
585
600
|
if bad:
|
|
586
|
-
|
|
601
|
+
raise ValueError(
|
|
602
|
+
f"patched linear peak map differs from full recompute at {bad} sites"
|
|
603
|
+
)
|
|
587
604
|
|
|
588
605
|
|
|
589
606
|
def find_score_net(pipeline: "WaveformPipeline"):
|
|
@@ -596,6 +613,7 @@ def find_score_net(pipeline: "WaveformPipeline"):
|
|
|
596
613
|
|
|
597
614
|
def make_peak_proposer(
|
|
598
615
|
p: SubtractionConfig,
|
|
616
|
+
proposal: DetectionProposal,
|
|
599
617
|
spike_length_samples: int,
|
|
600
618
|
trough_offset_samples: int,
|
|
601
619
|
peak_channel_index: Tensor | None,
|
|
@@ -606,41 +624,16 @@ def make_peak_proposer(
|
|
|
606
624
|
proposal_filter_trough_offset: int | None = None,
|
|
607
625
|
audit: bool = False,
|
|
608
626
|
) -> PeakProposer:
|
|
609
|
-
if p.
|
|
610
|
-
panic(f"unknown {p.detection_proposal=}")
|
|
611
|
-
|
|
612
|
-
if p.detection_proposal in ("tpca", "vq") and proposal_filters is not None:
|
|
613
|
-
assert channel_index is not None
|
|
614
|
-
assert proposal_filter_trough_offset is not None
|
|
615
|
-
return LinearMatchedFilterProposer(
|
|
616
|
-
filters=proposal_filters,
|
|
617
|
-
filter_trough_offset=proposal_filter_trough_offset,
|
|
618
|
-
threshold=p.score_proposal_threshold**2,
|
|
619
|
-
reduction="sum" if p.detection_proposal == "tpca" else "max",
|
|
620
|
-
relative_peak_radius=p.relative_peak_radius_samples,
|
|
621
|
-
detect_dedup_radius=spike_length_samples,
|
|
622
|
-
peak_channel_index=peak_channel_index,
|
|
623
|
-
remove_exact_duplicates=p.remove_exact_duplicates,
|
|
624
|
-
trough_offset_samples=trough_offset_samples,
|
|
625
|
-
spike_length_samples=spike_length_samples,
|
|
626
|
-
field_patch_channel_index=channel_index,
|
|
627
|
-
audit=audit,
|
|
628
|
-
)
|
|
629
|
-
|
|
630
|
-
if p.detection_proposal == "score_net":
|
|
627
|
+
if p.propose_with_score_net:
|
|
631
628
|
assert denoising_pipeline is not None and channel_index is not None
|
|
632
629
|
denoiser, score_net = find_score_net(denoising_pipeline)
|
|
633
|
-
|
|
634
|
-
panic(
|
|
635
|
-
"detection_proposal='score_net' but the denoising pipeline has no "
|
|
636
|
-
"score net. Set score_filter_radius_um so one gets trained."
|
|
637
|
-
)
|
|
630
|
+
assert score_net is not None
|
|
638
631
|
score_channel_index = denoiser.b.score_channel_index
|
|
639
632
|
return MatchedFilterProposer(
|
|
640
633
|
score_net=score_net,
|
|
641
634
|
score_channel_index=score_channel_index,
|
|
642
635
|
score_mask=score_channel_index < denoiser.n_channels,
|
|
643
|
-
threshold=p.
|
|
636
|
+
threshold=p.score_net_threshold,
|
|
644
637
|
relative_peak_radius=p.relative_peak_radius_samples,
|
|
645
638
|
detect_dedup_radius=spike_length_samples,
|
|
646
639
|
peak_channel_index=peak_channel_index,
|
|
@@ -652,7 +645,8 @@ def make_peak_proposer(
|
|
|
652
645
|
),
|
|
653
646
|
audit=audit,
|
|
654
647
|
)
|
|
655
|
-
|
|
648
|
+
|
|
649
|
+
if proposal == "voltage" and p.subtract_global_dedup:
|
|
656
650
|
return GlobalPeakProposer(
|
|
657
651
|
threshold=p.voltage_threshold,
|
|
658
652
|
peak_sign=p.peak_sign,
|
|
@@ -665,16 +659,97 @@ def make_peak_proposer(
|
|
|
665
659
|
spike_length_samples=spike_length_samples,
|
|
666
660
|
audit=audit,
|
|
667
661
|
)
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
662
|
+
elif proposal == "voltage":
|
|
663
|
+
return LocalPeakProposer(
|
|
664
|
+
threshold=p.voltage_threshold,
|
|
665
|
+
peak_sign=p.peak_sign,
|
|
666
|
+
relative_peak_radius=p.relative_peak_radius_samples,
|
|
667
|
+
detect_dedup_radius=spike_length_samples,
|
|
668
|
+
peak_channel_index=peak_channel_index,
|
|
669
|
+
sub_dedup_channel_index=sub_dedup_channel_index,
|
|
670
|
+
trough_priority=p.trough_priority,
|
|
671
|
+
remove_exact_duplicates=p.remove_exact_duplicates,
|
|
672
|
+
)
|
|
673
|
+
elif proposal == "tpca" or proposal == "vq":
|
|
674
|
+
assert proposal_filters is not None
|
|
675
|
+
assert channel_index is not None
|
|
676
|
+
assert proposal_filter_trough_offset is not None
|
|
677
|
+
return LinearMatchedFilterProposer(
|
|
678
|
+
filters=proposal_filters,
|
|
679
|
+
filter_trough_offset=proposal_filter_trough_offset,
|
|
680
|
+
threshold=p.proposal_threshold**2,
|
|
681
|
+
reduction="sum" if proposal == "tpca" else "max",
|
|
682
|
+
relative_peak_radius=p.relative_peak_radius_samples,
|
|
683
|
+
detect_dedup_radius=spike_length_samples,
|
|
684
|
+
peak_channel_index=peak_channel_index,
|
|
685
|
+
remove_exact_duplicates=p.remove_exact_duplicates,
|
|
686
|
+
trough_offset_samples=trough_offset_samples,
|
|
687
|
+
spike_length_samples=spike_length_samples,
|
|
688
|
+
field_patch_channel_index=channel_index,
|
|
689
|
+
audit=audit,
|
|
690
|
+
)
|
|
691
|
+
else:
|
|
692
|
+
panic(proposal)
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
def with_proposal_filters(
|
|
696
|
+
featurization_cfg: FeaturizationConfig, proposal: DetectionProposal, n_filters: int
|
|
697
|
+
) -> FeaturizationConfig:
|
|
698
|
+
"""Swap fields to make a featurization config whose pipeline will fit the proposer"""
|
|
699
|
+
if proposal == "voltage":
|
|
700
|
+
return featurization_cfg
|
|
701
|
+
elif proposal == "tpca":
|
|
702
|
+
if n_filters > featurization_cfg.tpca_rank:
|
|
703
|
+
panic(f"{n_filters} {featurization_cfg.tpca_rank}")
|
|
704
|
+
return replace(featurization_cfg, learn_cleaned_tpca_basis=True)
|
|
705
|
+
elif proposal == "vq":
|
|
706
|
+
return replace(featurization_cfg, vq_proposal_filters=n_filters)
|
|
707
|
+
else:
|
|
708
|
+
panic(proposal)
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def proposal_fit_featurization_cfg(
|
|
712
|
+
featurization_cfg: FeaturizationConfig | None,
|
|
713
|
+
proposal: DetectionProposal,
|
|
714
|
+
n_filters: int,
|
|
715
|
+
) -> FeaturizationConfig:
|
|
716
|
+
if featurization_cfg is None and proposal != "voltage":
|
|
717
|
+
raise ValueError(
|
|
718
|
+
f"{proposal=} needs the featurization config its filters are fit with."
|
|
719
|
+
)
|
|
720
|
+
if featurization_cfg is None:
|
|
721
|
+
featurization_cfg = FeaturizationConfig()
|
|
722
|
+
waveforms_only = FeaturizationConfig(
|
|
723
|
+
do_tpca_denoise=False,
|
|
724
|
+
do_enforce_decrease="no",
|
|
725
|
+
save_input_voltages=False,
|
|
726
|
+
save_input_waveforms=True,
|
|
727
|
+
save_input_tpca_projs=False,
|
|
728
|
+
save_amplitudes=False,
|
|
729
|
+
do_localization=False,
|
|
730
|
+
input_waveforms_name="",
|
|
731
|
+
tpca_rank=featurization_cfg.tpca_rank,
|
|
732
|
+
tpca_fit_radius=featurization_cfg.tpca_fit_radius,
|
|
733
|
+
tpca_max_waveforms=featurization_cfg.tpca_max_waveforms,
|
|
734
|
+
input_tpca_waveform_cfg=featurization_cfg.input_tpca_waveform_cfg,
|
|
677
735
|
)
|
|
736
|
+
return with_proposal_filters(waveforms_only, proposal, n_filters)
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
def proposal_filters_from_pipeline(
|
|
740
|
+
pipeline: "WaveformPipeline", proposal: DetectionProposal, n_filters: int
|
|
741
|
+
) -> tuple[Tensor, int]:
|
|
742
|
+
if proposal == "tpca":
|
|
743
|
+
nodes = [t for t in pipeline.transformers if type(t) is BaseTemporalPCA]
|
|
744
|
+
elif proposal == "vq":
|
|
745
|
+
nodes = [t for t in pipeline.transformers if isinstance(t, VQMatchedFilter)]
|
|
746
|
+
else:
|
|
747
|
+
panic()
|
|
748
|
+
if len(nodes) != 1:
|
|
749
|
+
panic(len(nodes))
|
|
750
|
+
(node,) = nodes
|
|
751
|
+
assert not node.needs_fit()
|
|
752
|
+
return node.b.components[:n_filters].clone(), node.filter_trough_offset
|
|
678
753
|
|
|
679
754
|
|
|
680
755
|
# -- the subtracter
|