DARTsort 0.5.13__tar.gz → 0.5.14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dartsort-0.5.13 → dartsort-0.5.14}/PKG-INFO +1 -1
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/mixture.py +3 -1
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/config.py +3 -3
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/comparison.py +6 -1
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/peel_base.py +11 -1
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/peel_lib.py +64 -32
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/_multichan_denoiser_kit.py +109 -34
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/amplitudes.py +13 -6
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/decollider.py +18 -8
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/pipeline.py +12 -6
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/internal_config.py +5 -4
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/main_util.py +2 -1
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/peel_util.py +16 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/PKG-INFO +1 -1
- {dartsort-0.5.13 → dartsort-0.5.14}/.github/workflows/ci.yml +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/.github/workflows/ci_windows.yml +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/.github/workflows/deploy.yml +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/.github/workflows/docs.yml +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/.gitignore +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/README.md +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/docs/index.md +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/docs/main_api.md +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/docs/toolkit_api.md +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/environment.yml +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/notebook/.keep +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/pyproject.toml +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/setup.cfg +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/cli.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/agglomerate.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/cluster_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/clustering.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/clustering_features.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/density.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/forward_backward.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/kmeans.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/merge.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/modes.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/clustering/refine_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/detect/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/detect/detect.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/detect/detect_filters.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/analysis.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/hybrid_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/noise_recording_tools.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/sim_template_tools.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/simkit.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/evaluate/simlib.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/localize/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/localize/localize_torch.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/localize/localize_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/main.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/grab.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/compressed_upsampled.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/drifty.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/matching_base.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/pairwise.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/matching_util/pairwise_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/reduction_template.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/shaver.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/subtract.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/peel/threshold.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/detect_np1.pt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/detect_np2.pt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/glasbey1024.npz +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/get_templates.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/postprocess_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/realignment.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/template_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/templates.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/templates/templib.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/all_transformers.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/amortized_localization.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/denoising_scorer.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/enforce_decrease.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/fixed_prop.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/interp.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/localize.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/matching_denoiser.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/mixture_classifier.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/reduction.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/single_channel_denoiser.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/supervised_denoiser.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/temporal_pca.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/transform_base.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/transform/whiten.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/cli_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/data_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/drift_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/interpolation_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/job_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/logging_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/more_operators.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/motion.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/multiprocessing_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/nn_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/noise_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/preprocess_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/py_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/registration_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/residual_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/sparse_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/spikeio.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/spiketorch.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/testing_util/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/testing_util/matching_debug_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/testing_util/mixture_testing_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/torch_optimization_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/torch_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/util/waveform_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/__init__.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/analysis_plots.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/colors.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/gt.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/layout.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/metrics_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/mixture.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/recanim.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/scatterplots.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/sorting.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/unit.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/unit_comparison.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/versus.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/vismain.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort/vis/waveforms.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/SOURCES.txt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/dependency_links.txt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/entry_points.txt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/requires.txt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/src/dartsort.egg-info/top_level.txt +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/conftest.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_agglomerate.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_alignment.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_cluster_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_clustering.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_config.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_dartsort.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_data_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_density.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_detect.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_drift_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_drifty_matching.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_enforce_decrease.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_grab_and_featurize.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_kmeans.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_linalg.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_localize_torch.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_matching.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_mixture_classifier.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_modeling.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_simkit.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_spiketorch.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_subtract.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_templates.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_threshold.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_torch_optimization_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_transform.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/tests/test_waveform_util.py +0 -0
- {dartsort-0.5.13 → dartsort-0.5.14}/zensical.toml +0 -0
|
@@ -2683,7 +2683,9 @@ class TruncatedMixtureModel(BaseMixtureModel):
|
|
|
2683
2683
|
# support of the likelihoods to drift away slightly from the weights',
|
|
2684
2684
|
# and this can lead to slightly negative elbo changes. giving a little
|
|
2685
2685
|
# more leeway than in em() here for when to panic about it.
|
|
2686
|
-
|
|
2686
|
+
# in fact, I'm going to comment this assertion out, but leaving the
|
|
2687
|
+
# comment here.
|
|
2688
|
+
# assert (len(elbos) < 2) or (elbos[-1] > elbos[0] - 5e-2)
|
|
2687
2689
|
|
|
2688
2690
|
def score(
|
|
2689
2691
|
self,
|
|
@@ -53,10 +53,10 @@ class DARTsortUserConfig:
|
|
|
53
53
|
that there is enough data for clustering. See also subsampling_presence.
|
|
54
54
|
Set to None to disable subsampling."""
|
|
55
55
|
|
|
56
|
-
subsampling_presence: Annotated[float, Field(gt=0.0, le=1.0)] = 0.
|
|
56
|
+
subsampling_presence: Annotated[float, Field(gt=0.0, le=1.0)] = 0.025
|
|
57
57
|
"""Early detection steps which have already found `subsampling_spikes`
|
|
58
58
|
spikes are only allowed to end early if they additionally cover this
|
|
59
|
-
fraction of the recording, to make sure there's
|
|
59
|
+
fraction of the recording, to make sure there's some coverage of
|
|
60
60
|
conditions for template estimation."""
|
|
61
61
|
|
|
62
62
|
matching_iterations: int = 1
|
|
@@ -379,7 +379,7 @@ class DeveloperConfig(DARTsortUserConfig):
|
|
|
379
379
|
# store extra intermediates@
|
|
380
380
|
save_subtracted_waveforms: bool = False
|
|
381
381
|
save_collisioncleaned_waveforms: bool = False
|
|
382
|
-
|
|
382
|
+
always_save_detailed_features: bool = False
|
|
383
383
|
precomputed_templates_npz: str | None = None
|
|
384
384
|
save_everything_on_error: bool = False
|
|
385
385
|
link_from: str | None = None
|
|
@@ -68,6 +68,7 @@ class DARTsortGroundTruthComparison:
|
|
|
68
68
|
self._unsorted_detection = self._greedy_confusion = None
|
|
69
69
|
if self.compute_unsorted_recall:
|
|
70
70
|
self._calculate_greedy_confusion_and_detection()
|
|
71
|
+
self._matched_and_missed_cache = {}
|
|
71
72
|
self._unit_info_cache = {}
|
|
72
73
|
|
|
73
74
|
def _check(self):
|
|
@@ -338,6 +339,8 @@ class DARTsortGroundTruthComparison:
|
|
|
338
339
|
assert self._unsorted_detection.shape == (gtns,)
|
|
339
340
|
|
|
340
341
|
def matched_and_missed(self, gt_unit):
|
|
342
|
+
if gt_unit in self._matched_and_missed_cache:
|
|
343
|
+
return self._matched_and_missed_cache[gt_unit]
|
|
341
344
|
(gt_spike_labels,) = self.comparison.get_labels1(gt_unit)
|
|
342
345
|
assert self.gt_analysis.sorting.labels is not None
|
|
343
346
|
in_gt_unit = self.gt_si_inds[
|
|
@@ -346,7 +349,9 @@ class DARTsortGroundTruthComparison:
|
|
|
346
349
|
matched_gt_mask = gt_spike_labels == "TP"
|
|
347
350
|
matched_gt_indices = in_gt_unit[matched_gt_mask]
|
|
348
351
|
only_gt_indices = in_gt_unit[np.logical_not(matched_gt_mask)]
|
|
349
|
-
|
|
352
|
+
ret = in_gt_unit, matched_gt_indices, only_gt_indices
|
|
353
|
+
self._matched_and_missed_cache[gt_unit] = ret
|
|
354
|
+
return ret
|
|
350
355
|
|
|
351
356
|
def get_spikes_by_category(self, gt_unit, tested_unit=None):
|
|
352
357
|
if tested_unit is None:
|
|
@@ -9,6 +9,7 @@ from pathlib import Path
|
|
|
9
9
|
from sys import getrefcount
|
|
10
10
|
from threading import Lock, local
|
|
11
11
|
from typing import Any, Sequence, TypedDict
|
|
12
|
+
from warnings import catch_warnings, filterwarnings
|
|
12
13
|
|
|
13
14
|
import h5py
|
|
14
15
|
import numpy as np
|
|
@@ -539,8 +540,17 @@ class BasePeeler(BModule):
|
|
|
539
540
|
channel_indices=None,
|
|
540
541
|
margin=self.chunk_margin_samples,
|
|
541
542
|
)
|
|
543
|
+
assert isinstance(chunk, np.ndarray)
|
|
542
544
|
device = self.b.channel_index.device
|
|
543
|
-
|
|
545
|
+
if device.type == "cpu":
|
|
546
|
+
chunk = torch.tensor(chunk, device=device, dtype=self.dtype)
|
|
547
|
+
else:
|
|
548
|
+
# we have to copy anyway to go to device, so working around the
|
|
549
|
+
# torch warning here. this way, can avoid blocking.
|
|
550
|
+
with catch_warnings():
|
|
551
|
+
filterwarnings("ignore", message="The given NumPy array is not ")
|
|
552
|
+
chunk = torch.from_numpy(chunk)
|
|
553
|
+
chunk = chunk.to(device=device, dtype=self.dtype, non_blocking=True)
|
|
544
554
|
return chunk, chunk_end_samples, left_margin, right_margin
|
|
545
555
|
|
|
546
556
|
def featurize_chunk_result(
|
|
@@ -20,6 +20,7 @@ from ..util.internal_config import (
|
|
|
20
20
|
)
|
|
21
21
|
from ..util.job_util import ensure_computation_config
|
|
22
22
|
from ..util.spiketorch import grab_spikes, ptp, subtract_spikes_
|
|
23
|
+
from ..util.torch_util import torch_compile
|
|
23
24
|
from ..util.waveform_util import get_relative_subset, make_channel_index
|
|
24
25
|
from .peel_base import PeelingBatchResult
|
|
25
26
|
|
|
@@ -89,46 +90,32 @@ def check_residual_decrease(
|
|
|
89
90
|
return None, {}
|
|
90
91
|
assert orig_wfs is not None
|
|
91
92
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
# spatial mul -- putting temporal dim last here
|
|
104
|
-
orig_wfs = W.bmm(orig_wfs.mT)
|
|
105
|
-
dn_wfs = W.bmm(dn_wfs.mT)
|
|
106
|
-
|
|
107
|
-
# temporal conv if needed
|
|
108
|
-
if whitening_kernel is not None:
|
|
109
|
-
*shp, t = orig_wfs.shape
|
|
110
|
-
k = whitening_kernel[None, None]
|
|
111
|
-
orig_wfs = F.conv1d(orig_wfs.view(-1, 1, t), k, padding="same")
|
|
112
|
-
dn_wfs = F.conv1d(dn_wfs.view(-1, 1, t), k, padding="same")
|
|
113
|
-
orig_wfs = orig_wfs.view(*shp, t)
|
|
114
|
-
dn_wfs = dn_wfs.view(*shp, t)
|
|
93
|
+
orig_wfs = flatten_denan_and_whiten_batched(
|
|
94
|
+
orig_wfs,
|
|
95
|
+
channels,
|
|
96
|
+
local_whiteners,
|
|
97
|
+
whitening_kernel,
|
|
98
|
+
overwrite=overwrite_orig_waveforms,
|
|
99
|
+
)
|
|
100
|
+
dn_wfs = flatten_denan_and_whiten_batched(
|
|
101
|
+
dn_wfs, channels, local_whiteners, whitening_kernel
|
|
102
|
+
)
|
|
115
103
|
|
|
116
104
|
if decrease_objective == "deconv":
|
|
117
105
|
if overwrite_orig_waveforms:
|
|
118
|
-
buf = orig_wfs.mul_(dn_wfs)
|
|
119
|
-
conv = buf.sum(dim=
|
|
106
|
+
buf = orig_wfs.mul_(dn_wfs)
|
|
107
|
+
conv = buf.sum(dim=1)
|
|
120
108
|
torch.square(dn_wfs, out=buf)
|
|
121
|
-
norm = buf.
|
|
109
|
+
norm = buf.sum(dim=1)
|
|
122
110
|
else:
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
norm = dn_wfs.square_().sum(dim=(1, 2))
|
|
111
|
+
conv = (orig_wfs * dn_wfs).sum(dim=1)
|
|
112
|
+
norm = dn_wfs.square_().sum(dim=1)
|
|
126
113
|
reduction = conv.mul_(2.0).sub_(norm)
|
|
127
114
|
threshold = threshold**2
|
|
128
115
|
elif decrease_objective in ("norm", "normsq"):
|
|
129
|
-
orig_decobj = orig_wfs.square().sum(dim=
|
|
130
|
-
orig_wfs = orig_wfs.sub_(dn_wfs)
|
|
131
|
-
new_decobj = orig_wfs.square_().sum(dim=
|
|
116
|
+
orig_decobj = orig_wfs.square().sum(dim=1)
|
|
117
|
+
orig_wfs = orig_wfs.sub_(dn_wfs)
|
|
118
|
+
new_decobj = orig_wfs.square_().sum(dim=1)
|
|
132
119
|
if decrease_objective == "norm":
|
|
133
120
|
orig_decobj = orig_decobj.sqrt_()
|
|
134
121
|
new_decobj = new_decobj.sqrt_()
|
|
@@ -147,6 +134,51 @@ def check_residual_decrease(
|
|
|
147
134
|
return keep, features
|
|
148
135
|
|
|
149
136
|
|
|
137
|
+
@torch_compile
|
|
138
|
+
def flatten_denan_and_whiten_batched(
|
|
139
|
+
wfs: torch.Tensor,
|
|
140
|
+
channels: torch.Tensor | None,
|
|
141
|
+
local_whiteners: Tensor | None,
|
|
142
|
+
whitening_kernel: Tensor | None,
|
|
143
|
+
overwrite: bool = False,
|
|
144
|
+
batch_size: int = 4096,
|
|
145
|
+
):
|
|
146
|
+
N = wfs.shape[0]
|
|
147
|
+
T = wfs.shape[1]
|
|
148
|
+
C = wfs.shape[2]
|
|
149
|
+
if overwrite:
|
|
150
|
+
wfs = wfs.nan_to_num_()
|
|
151
|
+
else:
|
|
152
|
+
wfs = wfs.nan_to_num()
|
|
153
|
+
|
|
154
|
+
if local_whiteners is None:
|
|
155
|
+
return wfs.view(N, -1)
|
|
156
|
+
|
|
157
|
+
assert channels is not None
|
|
158
|
+
|
|
159
|
+
W = local_whiteners[channels]
|
|
160
|
+
wfs = W.bmm(wfs.mT)
|
|
161
|
+
if whitening_kernel is None:
|
|
162
|
+
return wfs.view(N, -1)
|
|
163
|
+
|
|
164
|
+
k = whitening_kernel[None, None]
|
|
165
|
+
NC = N * C
|
|
166
|
+
wfs = wfs.view(NC, 1, T)
|
|
167
|
+
|
|
168
|
+
for i0 in range(0, NC, batch_size):
|
|
169
|
+
i1 = min(NC, i0 + batch_size)
|
|
170
|
+
nb = i1 - i0
|
|
171
|
+
y = wfs[i0:i1]
|
|
172
|
+
if nb < batch_size:
|
|
173
|
+
y = F.pad(y, (0, 0, 0, 0, 0, batch_size - nb))
|
|
174
|
+
z = F.conv1d(y, k, padding="same")
|
|
175
|
+
if nb < batch_size:
|
|
176
|
+
z = z[:nb]
|
|
177
|
+
wfs[i0:i1] = z
|
|
178
|
+
|
|
179
|
+
return wfs.view(N, -1)
|
|
180
|
+
|
|
181
|
+
|
|
150
182
|
ChunkSubtractionResult = namedtuple(
|
|
151
183
|
"ChunkSubtractionResult",
|
|
152
184
|
[
|
|
@@ -1,14 +1,15 @@
|
|
|
1
|
-
from queue import Queue
|
|
2
|
-
from threading import Thread
|
|
1
|
+
from queue import Empty, Queue
|
|
2
|
+
from threading import Lock, Thread, local
|
|
3
3
|
|
|
4
4
|
import h5py
|
|
5
|
+
import numba
|
|
5
6
|
import numpy as np
|
|
6
7
|
import torch
|
|
7
8
|
from torch.utils.data import DataLoader, Dataset, Sampler
|
|
8
9
|
|
|
9
10
|
from ..util import nn_util, spikeio
|
|
10
11
|
from ..util.logging_util import get_logger
|
|
11
|
-
from ..util.spiketorch import get_relative_index, reindex
|
|
12
|
+
from ..util.spiketorch import get_relative_index, reindex, spawn_torch_rg
|
|
12
13
|
from ..util.waveform_util import regularize_channel_index
|
|
13
14
|
from .transform_base import BaseWaveformDenoiser
|
|
14
15
|
|
|
@@ -308,12 +309,17 @@ def get_noise_h5(
|
|
|
308
309
|
spike_length_samples=121,
|
|
309
310
|
rg: np.random.Generator,
|
|
310
311
|
count: int | None = None,
|
|
312
|
+
local=True,
|
|
311
313
|
):
|
|
312
314
|
if count is None:
|
|
313
315
|
count = dset.shape[0]
|
|
314
316
|
|
|
315
|
-
|
|
316
|
-
|
|
317
|
+
if local:
|
|
318
|
+
ii0 = rg.integers(count - len(channels)).item()
|
|
319
|
+
ii = slice(ii0, ii0 + len(channels))
|
|
320
|
+
else:
|
|
321
|
+
ii = rg.choice(count, size=len(channels), replace=False)
|
|
322
|
+
ii.sort()
|
|
317
323
|
|
|
318
324
|
if dset.shape[1] > spike_length_samples:
|
|
319
325
|
pad = dset.shape[1] - spike_length_samples
|
|
@@ -324,15 +330,24 @@ def get_noise_h5(
|
|
|
324
330
|
|
|
325
331
|
# channels...
|
|
326
332
|
cc = channel_index[channels]
|
|
327
|
-
valid = cc < channel_index.shape[0]
|
|
328
|
-
cc = np.where(valid, cc, 0)
|
|
329
333
|
targ_shp = (cc.shape[0], noise_waveforms.shape[1], cc.shape[1])
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
# mask out
|
|
333
|
-
noise_waveforms *= valid[:, None, :].astype(noise_waveforms.dtype)
|
|
334
|
+
out = np.empty_like(noise_waveforms, shape=targ_shp)
|
|
335
|
+
take_along_dim2_masked(noise_waveforms, out, cc, channel_index.shape[0])
|
|
334
336
|
|
|
335
|
-
return torch.from_numpy(
|
|
337
|
+
return torch.from_numpy(out)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
@numba.njit(parallel=True)
|
|
341
|
+
def take_along_dim2_masked(arr, out, channels, mask_val):
|
|
342
|
+
for i in numba.prange(arr.shape[0]): # type: ignore
|
|
343
|
+
wf = arr[i]
|
|
344
|
+
cc = channels[i]
|
|
345
|
+
for t in range(arr.shape[1]):
|
|
346
|
+
for j, c in enumerate(cc):
|
|
347
|
+
if c == mask_val:
|
|
348
|
+
out[i, t, j] = 0.0
|
|
349
|
+
else:
|
|
350
|
+
out[i, t, j] = wf[t, c]
|
|
336
351
|
|
|
337
352
|
|
|
338
353
|
class NoneDataset(Dataset):
|
|
@@ -519,19 +534,29 @@ class AsyncBatchDataset(RefreshableDataset):
|
|
|
519
534
|
n_examples,
|
|
520
535
|
spike_length_samples=121,
|
|
521
536
|
generator=None,
|
|
537
|
+
rg=None,
|
|
522
538
|
chunk_size=2048,
|
|
523
539
|
queue_chunks=8,
|
|
540
|
+
n_workers=1,
|
|
524
541
|
):
|
|
525
542
|
super().__init__()
|
|
526
543
|
self.spike_length_samples = spike_length_samples
|
|
527
544
|
# note: although there's threading here, this is only used in one
|
|
528
545
|
# thread, so that's okay.
|
|
529
|
-
self.
|
|
546
|
+
self._generator = generator
|
|
547
|
+
self._rg = rg
|
|
530
548
|
self.n_examples = n_examples
|
|
549
|
+
self.n_workers = n_workers
|
|
550
|
+
|
|
551
|
+
self.lock = Lock()
|
|
552
|
+
self.locals = local()
|
|
553
|
+
self.locals.rg = None
|
|
554
|
+
self.locals.generator = None
|
|
531
555
|
|
|
532
556
|
self.chunk_size = chunk_size
|
|
533
557
|
self._queue = Queue(maxsize=queue_chunks)
|
|
534
|
-
self.
|
|
558
|
+
self._batches = Queue()
|
|
559
|
+
self._threads = None
|
|
535
560
|
self._indices = None
|
|
536
561
|
self._cur_data_ix = None
|
|
537
562
|
self._cur_chunk = None
|
|
@@ -543,7 +568,10 @@ class AsyncBatchDataset(RefreshableDataset):
|
|
|
543
568
|
|
|
544
569
|
def destroy(self):
|
|
545
570
|
self.bye = True
|
|
546
|
-
if self.
|
|
571
|
+
if self._batches is not None:
|
|
572
|
+
if hasattr(self._batches, "shutdown"):
|
|
573
|
+
self._batches.shutdown(True) # type: ignore
|
|
574
|
+
if self._threads is not None:
|
|
547
575
|
if hasattr(self._queue, "shutdown"):
|
|
548
576
|
self._queue.shutdown(True) # type: ignore
|
|
549
577
|
while True:
|
|
@@ -551,20 +579,26 @@ class AsyncBatchDataset(RefreshableDataset):
|
|
|
551
579
|
self._queue.get()
|
|
552
580
|
except Exception:
|
|
553
581
|
break
|
|
554
|
-
self.
|
|
555
|
-
|
|
556
|
-
|
|
582
|
+
for th in self._threads:
|
|
583
|
+
th.join()
|
|
584
|
+
for th in self._threads:
|
|
585
|
+
assert not th.is_alive()
|
|
586
|
+
self._threads = None
|
|
557
587
|
|
|
558
588
|
def cleanup(self):
|
|
559
|
-
if self.
|
|
560
|
-
self.
|
|
561
|
-
|
|
562
|
-
|
|
589
|
+
if self._threads is not None:
|
|
590
|
+
for th in self._threads:
|
|
591
|
+
th.join()
|
|
592
|
+
for th in self._threads:
|
|
593
|
+
assert not th.is_alive()
|
|
594
|
+
self._threads = None
|
|
563
595
|
|
|
564
596
|
def refresh(self, indices):
|
|
565
|
-
assert self.
|
|
597
|
+
assert self._threads is None, "Run cleanup()!"
|
|
566
598
|
self._cur_data_ix = 0
|
|
567
599
|
self._indices = indices
|
|
600
|
+
for batch in range(0, len(self._indices), self.chunk_size):
|
|
601
|
+
self._batches.put(batch)
|
|
568
602
|
self._run_thread()
|
|
569
603
|
|
|
570
604
|
def __getitem__(self, index):
|
|
@@ -572,8 +606,9 @@ class AsyncBatchDataset(RefreshableDataset):
|
|
|
572
606
|
assert self._cur_data_ix is not None
|
|
573
607
|
|
|
574
608
|
bs = len(index)
|
|
575
|
-
|
|
576
|
-
|
|
609
|
+
# if self.n_workers == 1:
|
|
610
|
+
# my_indices = self._indices[self._cur_data_ix : self._cur_data_ix + bs]
|
|
611
|
+
# assert torch.equal(torch.asarray(index), my_indices)
|
|
577
612
|
|
|
578
613
|
# need to batch up the chunks...
|
|
579
614
|
if self._cur_chunk is None:
|
|
@@ -594,19 +629,45 @@ class AsyncBatchDataset(RefreshableDataset):
|
|
|
594
629
|
self._cur_data_ix += bs
|
|
595
630
|
return noise_batch
|
|
596
631
|
|
|
632
|
+
def _initialize_locals(self):
|
|
633
|
+
if self._rg is not None and not hasattr(self.locals, "rg"):
|
|
634
|
+
with self.lock:
|
|
635
|
+
self.locals.rg = self._rg.spawn(1)[0]
|
|
636
|
+
if self._generator is not None and not hasattr(self.locals, "generator"):
|
|
637
|
+
with self.lock:
|
|
638
|
+
self.locals.generator = spawn_torch_rg(self._generator)
|
|
639
|
+
|
|
640
|
+
@property
|
|
641
|
+
def rg(self):
|
|
642
|
+
return self.locals.rg
|
|
643
|
+
|
|
644
|
+
@property
|
|
645
|
+
def generator(self):
|
|
646
|
+
return self.locals.generator
|
|
647
|
+
|
|
597
648
|
def _run_thread(self):
|
|
598
|
-
self.
|
|
599
|
-
|
|
649
|
+
self._threads = [
|
|
650
|
+
Thread(target=self._thread_main, daemon=True) for _ in range(self.n_workers)
|
|
651
|
+
]
|
|
652
|
+
for th in self._threads:
|
|
653
|
+
th.start()
|
|
600
654
|
|
|
601
655
|
def _thread_main(self):
|
|
602
656
|
assert self._indices is not None
|
|
603
|
-
|
|
604
|
-
|
|
657
|
+
assert self._batches is not None
|
|
658
|
+
self._initialize_locals()
|
|
659
|
+
while True:
|
|
660
|
+
try:
|
|
661
|
+
chunk_start = self._batches.get(timeout=0.01)
|
|
662
|
+
|
|
663
|
+
chunk_end = min(len(self._indices), chunk_start + self.chunk_size)
|
|
605
664
|
|
|
606
|
-
|
|
607
|
-
|
|
665
|
+
index = self._indices[chunk_start:chunk_end]
|
|
666
|
+
noise = self.load_batch(index)
|
|
608
667
|
|
|
609
|
-
|
|
668
|
+
self._queue.put(noise)
|
|
669
|
+
except Empty:
|
|
670
|
+
break
|
|
610
671
|
if self.bye:
|
|
611
672
|
break
|
|
612
673
|
|
|
@@ -624,15 +685,19 @@ class AsyncSameChannelNoiseDataset(AsyncBatchDataset):
|
|
|
624
685
|
channels,
|
|
625
686
|
spike_length_samples=121,
|
|
626
687
|
generator=None,
|
|
688
|
+
rg=None,
|
|
627
689
|
chunk_size=2048,
|
|
628
690
|
queue_chunks=8,
|
|
691
|
+
n_workers=1,
|
|
629
692
|
):
|
|
630
693
|
super().__init__(
|
|
631
694
|
n_examples=len(channels) if channels is not None else torch.inf,
|
|
632
695
|
spike_length_samples=spike_length_samples,
|
|
633
696
|
generator=generator,
|
|
697
|
+
rg=rg,
|
|
634
698
|
chunk_size=chunk_size,
|
|
635
699
|
queue_chunks=queue_chunks,
|
|
700
|
+
n_workers=n_workers,
|
|
636
701
|
)
|
|
637
702
|
self.channels = channels
|
|
638
703
|
self.channel_index = channel_index
|
|
@@ -654,9 +719,11 @@ class AsyncSameChannelHDF5NoiseDataset(AsyncSameChannelNoiseDataset):
|
|
|
654
719
|
generator=None,
|
|
655
720
|
chunk_size=256,
|
|
656
721
|
queue_chunks=8,
|
|
722
|
+
n_workers=1,
|
|
657
723
|
dataset_name="residual",
|
|
658
724
|
count_name="n_residuals",
|
|
659
725
|
memmap=True,
|
|
726
|
+
memory=False,
|
|
660
727
|
):
|
|
661
728
|
super().__init__(
|
|
662
729
|
channels=channels,
|
|
@@ -664,9 +731,16 @@ class AsyncSameChannelHDF5NoiseDataset(AsyncSameChannelNoiseDataset):
|
|
|
664
731
|
spike_length_samples=spike_length_samples,
|
|
665
732
|
generator=generator,
|
|
666
733
|
chunk_size=chunk_size,
|
|
734
|
+
rg=rg,
|
|
667
735
|
queue_chunks=queue_chunks,
|
|
736
|
+
n_workers=n_workers,
|
|
668
737
|
)
|
|
669
|
-
if
|
|
738
|
+
if memory:
|
|
739
|
+
with h5py.File(hdf5_filename, "r", locking=False) as h5:
|
|
740
|
+
self.count = h5[count_name][()]
|
|
741
|
+
self.dset = h5[dataset_name][: self.count]
|
|
742
|
+
self.h5 = None
|
|
743
|
+
elif memmap:
|
|
670
744
|
# this only works with contiguous datasets
|
|
671
745
|
# hacky but h5 reading is so slow... and this is so fast...
|
|
672
746
|
with h5py.File(hdf5_filename, "r", locking=False) as h5:
|
|
@@ -688,7 +762,6 @@ class AsyncSameChannelHDF5NoiseDataset(AsyncSameChannelNoiseDataset):
|
|
|
688
762
|
)
|
|
689
763
|
self.dset = self.h5[dataset_name]
|
|
690
764
|
self.count = self.h5[count_name][()]
|
|
691
|
-
self.rg = rg
|
|
692
765
|
assert self.dset.shape[1] >= spike_length_samples
|
|
693
766
|
assert self.dset.shape[2] == channel_index.shape[0]
|
|
694
767
|
|
|
@@ -719,6 +792,7 @@ class AsyncSameChannelRecordingNoiseDataset(AsyncSameChannelNoiseDataset):
|
|
|
719
792
|
generator=None,
|
|
720
793
|
chunk_size=2048,
|
|
721
794
|
queue_chunks=8,
|
|
795
|
+
n_workers=1,
|
|
722
796
|
):
|
|
723
797
|
super().__init__(
|
|
724
798
|
channels=channels,
|
|
@@ -727,6 +801,7 @@ class AsyncSameChannelRecordingNoiseDataset(AsyncSameChannelNoiseDataset):
|
|
|
727
801
|
generator=generator,
|
|
728
802
|
chunk_size=chunk_size,
|
|
729
803
|
queue_chunks=queue_chunks,
|
|
804
|
+
n_workers=n_workers,
|
|
730
805
|
)
|
|
731
806
|
self.recording = recording
|
|
732
807
|
|
|
@@ -72,6 +72,9 @@ class AmplitudeFeatures(BaseWaveformFeaturizer):
|
|
|
72
72
|
ptp_max_amplitude or ptp_amplitude_vectors or log_peak_to_trough
|
|
73
73
|
)
|
|
74
74
|
self.compute_maxchan = ptp_max_amplitude or log_peak_to_trough
|
|
75
|
+
self.just_maxptp = ptp_max_amplitude and not (
|
|
76
|
+
log_peak_to_trough or ptp_amplitude_vectors or peak_amplitude_vectors
|
|
77
|
+
)
|
|
75
78
|
|
|
76
79
|
super().__init__(name=names)
|
|
77
80
|
|
|
@@ -83,11 +86,16 @@ class AmplitudeFeatures(BaseWaveformFeaturizer):
|
|
|
83
86
|
max_vectors = min_vectors = ptp_vectors = None
|
|
84
87
|
maxs = mins = ptps = None
|
|
85
88
|
if self.compute_minmax_vectors:
|
|
86
|
-
max_vectors = waveforms.
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
89
|
+
min_vectors, max_vectors = waveforms.aminmax(dim=1)
|
|
90
|
+
if self.just_maxptp:
|
|
91
|
+
assert min_vectors is not None
|
|
92
|
+
assert max_vectors is not None
|
|
93
|
+
ptp_vectors = max_vectors.sub_(min_vectors)
|
|
94
|
+
ptps = ptp_vectors.nan_to_num_().amax(dim=1)
|
|
95
|
+
elif self.compute_maxchan:
|
|
96
|
+
assert min_vectors is not None
|
|
97
|
+
assert max_vectors is not None
|
|
98
|
+
ptp_vectors = max_vectors.sub(min_vectors)
|
|
91
99
|
maxchans = torch.argmax(ptp_vectors.nan_to_num(), dim=1, keepdim=True)
|
|
92
100
|
if self.log_peak_to_trough:
|
|
93
101
|
assert max_vectors is not None
|
|
@@ -98,7 +106,6 @@ class AmplitudeFeatures(BaseWaveformFeaturizer):
|
|
|
98
106
|
ptps = maxs - mins
|
|
99
107
|
elif self.ptp_max_amplitude:
|
|
100
108
|
ptps = torch.take_along_dim(ptp_vectors, maxchans, dim=1)[:, 0]
|
|
101
|
-
|
|
102
109
|
if self.ptp_amplitude_vectors:
|
|
103
110
|
features[self.ptp_amplitude_vectors_name] = ptp_vectors
|
|
104
111
|
if self.ptp_max_amplitude:
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import dataclasses
|
|
2
|
+
from math import isfinite
|
|
2
3
|
from typing import cast
|
|
3
4
|
|
|
4
5
|
import h5py
|
|
@@ -6,13 +7,10 @@ import numpy as np
|
|
|
6
7
|
import pandas as pd
|
|
7
8
|
import torch
|
|
8
9
|
import torch.nn.functional as F
|
|
9
|
-
from torch.utils.data import
|
|
10
|
-
DataLoader,
|
|
11
|
-
StackDataset,
|
|
12
|
-
TensorDataset,
|
|
13
|
-
)
|
|
10
|
+
from torch.utils.data import DataLoader, StackDataset, TensorDataset
|
|
14
11
|
|
|
15
12
|
from ..util.logging_util import get_logger, progrange
|
|
13
|
+
from ..util.multiprocessing_util import handle_negative_jobs
|
|
16
14
|
from ..util.spiketorch import reindex, spawn_torch_rg
|
|
17
15
|
from ._multichan_denoiser_kit import (
|
|
18
16
|
AOTIndicesWeightedRandomBatchSampler,
|
|
@@ -43,6 +41,8 @@ class Decollider(BaseMultichannelDenoiser):
|
|
|
43
41
|
name=None,
|
|
44
42
|
name_prefix="",
|
|
45
43
|
batch_size=512,
|
|
44
|
+
batches_per_chunk=4,
|
|
45
|
+
n_data_workers=1,
|
|
46
46
|
learning_rate=2e-4,
|
|
47
47
|
weight_decay=0.0,
|
|
48
48
|
n_epochs=100,
|
|
@@ -127,6 +127,8 @@ class Decollider(BaseMultichannelDenoiser):
|
|
|
127
127
|
warmup_lr=warmup_lr,
|
|
128
128
|
)
|
|
129
129
|
self.queue_chunks = queue_chunks
|
|
130
|
+
self.batches_per_chunk = batches_per_chunk
|
|
131
|
+
self.n_data_workers = n_data_workers
|
|
130
132
|
|
|
131
133
|
self.inference_z_samples = inference_z_samples
|
|
132
134
|
self.detach_amortizer = detach_amortizer
|
|
@@ -433,6 +435,8 @@ class Decollider(BaseMultichannelDenoiser):
|
|
|
433
435
|
):
|
|
434
436
|
rg = np.random.default_rng(self.random_seed)
|
|
435
437
|
|
|
438
|
+
_, n_data_workers = handle_negative_jobs(self.n_data_workers)
|
|
439
|
+
|
|
436
440
|
val_size = 0
|
|
437
441
|
train_indices = slice(None)
|
|
438
442
|
val_indices = None
|
|
@@ -463,7 +467,8 @@ class Decollider(BaseMultichannelDenoiser):
|
|
|
463
467
|
rg=np.random.default_rng(rg.spawn(1)[0]),
|
|
464
468
|
queue_chunks=self.queue_chunks,
|
|
465
469
|
dataset_name=dataset_name,
|
|
466
|
-
chunk_size=self.batch_size,
|
|
470
|
+
chunk_size=self.batches_per_chunk * self.batch_size,
|
|
471
|
+
n_workers=n_data_workers,
|
|
467
472
|
)
|
|
468
473
|
else:
|
|
469
474
|
logger.dartsortdebug("Load Decollider train noise data from recording.")
|
|
@@ -474,6 +479,7 @@ class Decollider(BaseMultichannelDenoiser):
|
|
|
474
479
|
spike_length_samples=spike_length_samples,
|
|
475
480
|
generator=spawn_torch_rg(rg),
|
|
476
481
|
queue_chunks=self.queue_chunks,
|
|
482
|
+
n_workers=n_data_workers,
|
|
477
483
|
)
|
|
478
484
|
if self.cycle_loss_alpha and can_load_h5:
|
|
479
485
|
logger.dartsortdebug(
|
|
@@ -488,6 +494,7 @@ class Decollider(BaseMultichannelDenoiser):
|
|
|
488
494
|
queue_chunks=self.queue_chunks,
|
|
489
495
|
dataset_name=dataset_name,
|
|
490
496
|
chunk_size=self.batch_size,
|
|
497
|
+
n_workers=n_data_workers,
|
|
491
498
|
)
|
|
492
499
|
elif self.cycle_loss_alpha:
|
|
493
500
|
logger.dartsortdebug("Load Decollider cycle noise data from recording.")
|
|
@@ -498,6 +505,7 @@ class Decollider(BaseMultichannelDenoiser):
|
|
|
498
505
|
spike_length_samples=spike_length_samples,
|
|
499
506
|
generator=spawn_torch_rg(rg),
|
|
500
507
|
queue_chunks=self.queue_chunks,
|
|
508
|
+
n_workers=n_data_workers,
|
|
501
509
|
)
|
|
502
510
|
else:
|
|
503
511
|
train_cycle_noise_dataset = NoneDataset(len(train_channels))
|
|
@@ -556,8 +564,8 @@ class Decollider(BaseMultichannelDenoiser):
|
|
|
556
564
|
# val set does not need shuffling
|
|
557
565
|
val_loader = DataLoader(
|
|
558
566
|
val_dataset,
|
|
559
|
-
num_workers=
|
|
560
|
-
persistent_workers=bool(
|
|
567
|
+
num_workers=n_data_workers, # type: ignore # ty: ignore[x]
|
|
568
|
+
persistent_workers=bool(n_data_workers),
|
|
561
569
|
batch_size=self.batch_size,
|
|
562
570
|
)
|
|
563
571
|
val_data = DecolliderDataLoader(
|
|
@@ -652,6 +660,8 @@ class Decollider(BaseMultichannelDenoiser):
|
|
|
652
660
|
train_losses = {
|
|
653
661
|
k: v.item() / len(train_data) for k, v in train_losses.items()
|
|
654
662
|
}
|
|
663
|
+
if not all(isfinite(v) for v in train_losses.values()):
|
|
664
|
+
raise ValueError(f"Denoiser training diverged: {train_losses}.")
|
|
655
665
|
train_records.append({**train_losses})
|
|
656
666
|
|
|
657
667
|
# Validation phase (only if val_loader is not None)
|
|
@@ -616,13 +616,19 @@ def _add_localization_and_ampvec(fc):
|
|
|
616
616
|
more = []
|
|
617
617
|
|
|
618
618
|
do_ptp_amp = do_feats and fc.save_amplitudes
|
|
619
|
-
do_peak_vec =
|
|
620
|
-
|
|
621
|
-
and fc.
|
|
622
|
-
and
|
|
619
|
+
do_peak_vec = (
|
|
620
|
+
do_feats
|
|
621
|
+
and fc.do_localization
|
|
622
|
+
and (fc.localization_amplitude_type == "peak")
|
|
623
|
+
and (fc.save_amplitude_vectors or not fc.nn_localization)
|
|
624
|
+
)
|
|
625
|
+
do_ptp_vec = (
|
|
626
|
+
do_feats
|
|
627
|
+
and fc.do_localization
|
|
628
|
+
and (fc.localization_amplitude_type == "ptp")
|
|
629
|
+
and (fc.save_amplitude_vectors or not fc.nn_localization)
|
|
623
630
|
)
|
|
624
|
-
|
|
625
|
-
do_logptt = do_feats and fc.save_amplitudes
|
|
631
|
+
do_logptt = do_feats and fc.save_all_amplitudes
|
|
626
632
|
do_any_amp = do_peak_vec or do_ptp_vec or do_ptp_amp or do_logptt
|
|
627
633
|
if do_any_amp or (do_feats and fc.save_all_amplitudes):
|
|
628
634
|
more.append(
|