DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from .internal_config import ComputationConfig, default_computation_cfg
|
|
2
|
+
|
|
3
|
+
comp_cfg_holder = {}
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def set_global_computation_config(comp_cfg: ComputationConfig):
|
|
7
|
+
global comp_cfg_holder
|
|
8
|
+
comp_cfg_holder["comp_cfg"] = comp_cfg
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def get_global_computation_config() -> ComputationConfig:
|
|
12
|
+
global comp_cfg_holder
|
|
13
|
+
return comp_cfg_holder.get("comp_cfg", default_computation_cfg)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def ensure_computation_config(cfg: ComputationConfig | None) -> ComputationConfig:
|
|
17
|
+
if cfg is None:
|
|
18
|
+
cfg = get_global_computation_config()
|
|
19
|
+
return cfg
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from logging import (
|
|
3
|
+
getLoggerClass,
|
|
4
|
+
addLevelName,
|
|
5
|
+
setLoggerClass,
|
|
6
|
+
NOTSET,
|
|
7
|
+
DEBUG,
|
|
8
|
+
getLogger,
|
|
9
|
+
getLevelNamesMapping,
|
|
10
|
+
basicConfig,
|
|
11
|
+
)
|
|
12
|
+
import warnings
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
DARTSORTVERBOSE = DEBUG + 4
|
|
16
|
+
addLevelName(DARTSORTVERBOSE, "DSVERBOSE")
|
|
17
|
+
|
|
18
|
+
DARTSORTDEBUG = DEBUG + 5
|
|
19
|
+
addLevelName(DARTSORTDEBUG, "DSDEBUG")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
DOUBLECHECK = DEBUG + 6
|
|
23
|
+
addLevelName(DOUBLECHECK, "DOUBLECHECK")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class DARTsortLogger(getLoggerClass()):
|
|
27
|
+
def __init__(self, name, level=NOTSET):
|
|
28
|
+
super().__init__(name, level)
|
|
29
|
+
|
|
30
|
+
def doublecheck(self, msg, *args, **kwargs):
|
|
31
|
+
if self.isEnabledFor(DOUBLECHECK):
|
|
32
|
+
self._log(DOUBLECHECK, msg, args, stacklevel=2, **kwargs)
|
|
33
|
+
|
|
34
|
+
def dartsortverbose(self, msg, *args, **kwargs):
|
|
35
|
+
if self.isEnabledFor(DARTSORTVERBOSE):
|
|
36
|
+
self._log(DARTSORTVERBOSE, msg, args, stacklevel=2, **kwargs)
|
|
37
|
+
|
|
38
|
+
def dartsortdebug(self, msg, *args, **kwargs):
|
|
39
|
+
if self.isEnabledFor(DARTSORTDEBUG):
|
|
40
|
+
self._log(DARTSORTDEBUG, msg, args, stacklevel=2, **kwargs)
|
|
41
|
+
|
|
42
|
+
def dartsortdebugthunk(self, msg, *args, **kwargs):
|
|
43
|
+
if self.isEnabledFor(DARTSORTDEBUG):
|
|
44
|
+
self._log(DARTSORTDEBUG, msg(), args, stacklevel=2, **kwargs)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
setLoggerClass(DARTsortLogger)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
logger = getLogger(__name__)
|
|
51
|
+
assert isinstance(logger, DARTsortLogger)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# set to environment-defined log level if present
|
|
55
|
+
if "LOG_LEVEL" in os.environ:
|
|
56
|
+
level = os.environ["LOG_LEVEL"]
|
|
57
|
+
try:
|
|
58
|
+
basicConfig(level=level)
|
|
59
|
+
except ValueError:
|
|
60
|
+
ilevel = int(level)
|
|
61
|
+
basicConfig(level=ilevel)
|
|
62
|
+
else:
|
|
63
|
+
ilevel = getLevelNamesMapping()[level]
|
|
64
|
+
logger.log(ilevel, f"Log level set to {level} ({ilevel}).")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def warn_with_traceback(message, category, filename, lineno, file=None, line=None):
|
|
68
|
+
import sys, traceback
|
|
69
|
+
|
|
70
|
+
log = file if hasattr(file, "write") else sys.stderr
|
|
71
|
+
assert log is not None
|
|
72
|
+
traceback.print_stack(file=log)
|
|
73
|
+
log.write(warnings.formatwarning(message, category, filename, lineno, line))
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# override warnings to show tracebacks when debugging
|
|
77
|
+
if logger.isEnabledFor(DARTSORTVERBOSE):
|
|
78
|
+
logger.dartsortdebug("Setting warnings.showwarning to print tracebacks.")
|
|
79
|
+
warnings.showwarning = warn_with_traceback
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def get_logger(*args, **kwargs) -> DARTsortLogger:
|
|
83
|
+
logger = getLogger(*args, **kwargs)
|
|
84
|
+
assert isinstance(logger, DARTsortLogger)
|
|
85
|
+
return logger
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
logger.dartsortdebug(
|
|
89
|
+
f"Logger is enabled for: DARTSORTDEBUG={logger.isEnabledFor(DARTSORTDEBUG)}, "
|
|
90
|
+
f"DARTSORTVERBOSE={logger.isEnabledFor(DARTSORTVERBOSE)}."
|
|
91
|
+
)
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
import shutil
|
|
2
|
+
from dataclasses import asdict
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from spikeinterface.core import BaseRecording
|
|
7
|
+
|
|
8
|
+
from ..util.data_util import DARTsortSorting
|
|
9
|
+
from ..util.internal_config import DARTsortInternalConfig
|
|
10
|
+
from ..util.logging_util import get_logger
|
|
11
|
+
from ..util.py_util import dartcopy2, dartcopytree, resolve_path
|
|
12
|
+
from ..util.registration_util import save_motion_est, try_load_motion_est
|
|
13
|
+
|
|
14
|
+
logger = get_logger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def ds_save_intermediate_labels(
|
|
18
|
+
step_name: str,
|
|
19
|
+
step_sorting: DARTsortSorting,
|
|
20
|
+
output_dir: Path | str | None,
|
|
21
|
+
cfg: DARTsortInternalConfig | None,
|
|
22
|
+
step_labels: np.ndarray | None = None,
|
|
23
|
+
work_dir: str | Path | None = None,
|
|
24
|
+
):
|
|
25
|
+
if cfg is not None and not cfg.save_intermediate_labels:
|
|
26
|
+
return
|
|
27
|
+
if output_dir is None:
|
|
28
|
+
return
|
|
29
|
+
output_dir = resolve_path(output_dir, strict=True)
|
|
30
|
+
if work_dir is None:
|
|
31
|
+
store_dir = output_dir
|
|
32
|
+
else:
|
|
33
|
+
store_dir = resolve_path(work_dir, strict=True)
|
|
34
|
+
|
|
35
|
+
step_labels_npy = store_dir / f"{step_name}_labels.npy"
|
|
36
|
+
logger.info(f"Saving {step_name} labels to {step_labels_npy}")
|
|
37
|
+
logger.info(f"{step_name}: {step_sorting}.")
|
|
38
|
+
if step_labels is None:
|
|
39
|
+
step_labels = step_sorting.labels
|
|
40
|
+
if step_labels is None:
|
|
41
|
+
raise ValueError(f"No step labels to save at step {step_name}.")
|
|
42
|
+
np.save(step_labels_npy, step_labels, allow_pickle=False)
|
|
43
|
+
|
|
44
|
+
if work_dir is not None:
|
|
45
|
+
targ_labels_npy = output_dir / step_labels_npy.name
|
|
46
|
+
logger.info(f"Copy {step_labels_npy} -> {targ_labels_npy}.")
|
|
47
|
+
dartcopy2(cfg, step_labels_npy, targ_labels_npy)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def ds_dump_config(internal_cfg: DARTsortInternalConfig, output_dir: Path):
|
|
51
|
+
import json
|
|
52
|
+
|
|
53
|
+
json_path = output_dir / "_dartsort_internal_config.json"
|
|
54
|
+
with open(json_path, "w") as jsonf:
|
|
55
|
+
json.dump(asdict(internal_cfg), jsonf)
|
|
56
|
+
logger.info(f"Recorded config to {json_path}.")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def ds_all_to_workdir(
|
|
60
|
+
*,
|
|
61
|
+
internal_cfg: DARTsortInternalConfig,
|
|
62
|
+
output_dir: Path,
|
|
63
|
+
work_dir: Path | None = None,
|
|
64
|
+
recording: BaseRecording,
|
|
65
|
+
overwrite=False,
|
|
66
|
+
rec_subdir="recppx",
|
|
67
|
+
sort_subdir="dartsort",
|
|
68
|
+
) -> tuple[BaseRecording, Path | None]:
|
|
69
|
+
"""Copy stuff to temporary working directory, if there is one."""
|
|
70
|
+
if work_dir is None:
|
|
71
|
+
return recording, None
|
|
72
|
+
|
|
73
|
+
if recording is not None and internal_cfg.copy_recording_to_tmpdir:
|
|
74
|
+
rec_dir = work_dir / rec_subdir
|
|
75
|
+
logger.info(
|
|
76
|
+
f"Writing recording to {rec_dir}. Parallelism is handled by "
|
|
77
|
+
"spikeinterface for this part, so use its `set_global_job_kwargs()` "
|
|
78
|
+
"function if you're waiting around."
|
|
79
|
+
)
|
|
80
|
+
recording = recording.save_to_folder(str(rec_dir))
|
|
81
|
+
|
|
82
|
+
sort_dir = work_dir / sort_subdir
|
|
83
|
+
if overwrite:
|
|
84
|
+
logger.info(f"Working in {work_dir}. No copy since {overwrite=}.")
|
|
85
|
+
return recording, sort_dir
|
|
86
|
+
|
|
87
|
+
# TODO: maybe no need to copy everything, esp. if fast forwarding?
|
|
88
|
+
logger.dartsortdebug(f"Copy {output_dir=} -> {sort_dir=}.")
|
|
89
|
+
dartcopytree(internal_cfg, output_dir, sort_dir)
|
|
90
|
+
return recording, sort_dir
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def ds_save_motion_est(
|
|
94
|
+
motion_est, output_dir: Path, work_dir: Path | None = None, overwrite: bool = False
|
|
95
|
+
):
|
|
96
|
+
if work_dir is None:
|
|
97
|
+
return
|
|
98
|
+
save_motion_est(motion_est, output_dir, overwrite=overwrite)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def ds_save_features(
|
|
102
|
+
cfg: DARTsortInternalConfig,
|
|
103
|
+
sorting: DARTsortSorting,
|
|
104
|
+
output_dir: Path,
|
|
105
|
+
work_dir: Path | None = None,
|
|
106
|
+
is_final=False,
|
|
107
|
+
):
|
|
108
|
+
if work_dir is None:
|
|
109
|
+
# nothing to copy
|
|
110
|
+
return
|
|
111
|
+
if not (cfg.save_intermediate_features or is_final):
|
|
112
|
+
return
|
|
113
|
+
|
|
114
|
+
# find h5 and models and copy
|
|
115
|
+
assert sorting.parent_h5_path is not None
|
|
116
|
+
h5_path = resolve_path(sorting.parent_h5_path)
|
|
117
|
+
assert h5_path.exists()
|
|
118
|
+
models_path = h5_path.parent / f"{h5_path.stem}_models"
|
|
119
|
+
|
|
120
|
+
targ_h5 = output_dir / h5_path.name
|
|
121
|
+
logger.dartsortdebug(f"Copy intermediate {h5_path=} -> {targ_h5=}.")
|
|
122
|
+
dartcopy2(cfg, h5_path, targ_h5)
|
|
123
|
+
|
|
124
|
+
if models_path.exists():
|
|
125
|
+
targ_models = output_dir / models_path.name
|
|
126
|
+
pconv_h5 = targ_models / "pconv.h5"
|
|
127
|
+
if cfg.matching_cfg.delete_pconv and pconv_h5.exists():
|
|
128
|
+
pconv_h5.unlink()
|
|
129
|
+
logger.dartsortdebug(f"Copy intermediate {models_path=} -> {targ_models=}.")
|
|
130
|
+
dartcopytree(cfg, models_path, targ_models)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def ds_handle_delete_intermediate_features(
|
|
134
|
+
cfg: DARTsortInternalConfig,
|
|
135
|
+
final_sorting: DARTsortSorting,
|
|
136
|
+
output_dir: Path,
|
|
137
|
+
work_dir: Path | None = None,
|
|
138
|
+
):
|
|
139
|
+
if work_dir is not None:
|
|
140
|
+
# they'll get deleted anyway and were not copied
|
|
141
|
+
return
|
|
142
|
+
if cfg.save_intermediate_features:
|
|
143
|
+
return
|
|
144
|
+
|
|
145
|
+
# find all non-final h5s, models and delete them
|
|
146
|
+
assert final_sorting.parent_h5_path is not None
|
|
147
|
+
final_h5 = resolve_path(final_sorting.parent_h5_path)
|
|
148
|
+
assert final_h5.exists()
|
|
149
|
+
assert final_h5.parent == output_dir
|
|
150
|
+
|
|
151
|
+
for h5_path in output_dir.glob("*.h5"):
|
|
152
|
+
if h5_path == final_h5:
|
|
153
|
+
continue
|
|
154
|
+
assert h5_path.name != final_h5.name
|
|
155
|
+
|
|
156
|
+
h5_path = output_dir / h5_path.name
|
|
157
|
+
models_path = output_dir / f"{h5_path.stem}_models"
|
|
158
|
+
|
|
159
|
+
logger.dartsortdebug(f"Clean up: remove {h5_path=}.")
|
|
160
|
+
h5_path.unlink()
|
|
161
|
+
if models_path.exists():
|
|
162
|
+
assert models_path.is_dir()
|
|
163
|
+
shutil.rmtree(models_path)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def ds_fast_forward(store_dir, cfg):
|
|
167
|
+
"""Fast-forward to the where sorting left off
|
|
168
|
+
|
|
169
|
+
# TODO: error if there is a saved cfg which differs? Maybe just optionally?
|
|
170
|
+
|
|
171
|
+
Returns
|
|
172
|
+
-------
|
|
173
|
+
next_step: int
|
|
174
|
+
cur_sorting: DARTsortSorting
|
|
175
|
+
"""
|
|
176
|
+
# this cur_h5 variable points to the peeling result we'll try to load
|
|
177
|
+
cur_h5 = sub_h5 = store_dir / "subtraction.h5"
|
|
178
|
+
cur_step = 0
|
|
179
|
+
if not sub_h5.exists():
|
|
180
|
+
return cur_step, None, None
|
|
181
|
+
|
|
182
|
+
# if subtraction is finished, we can try to load a motion estimate
|
|
183
|
+
motion_est = try_load_motion_est(store_dir)
|
|
184
|
+
mstr = ", and a loaded motion estimate" if motion_est is not None else ""
|
|
185
|
+
|
|
186
|
+
matching_h5s = sorted(store_dir.glob("matching*.h5"))
|
|
187
|
+
for cur_step, cur_h5 in enumerate(matching_h5s, start=1):
|
|
188
|
+
assert cur_h5.name == f"matching{cur_step}.h5"
|
|
189
|
+
|
|
190
|
+
# at this point, cur_h5 is the most recent h5 that exists, and
|
|
191
|
+
# current_step is the last step that was in progress.
|
|
192
|
+
# now, the peeling and clustering for that step may not have
|
|
193
|
+
# finished! in that case we want to resume from that step.
|
|
194
|
+
# on the other hand, the step could have finished entirely --
|
|
195
|
+
# but we would only know that if refined{step}_labels.npy exists,
|
|
196
|
+
# and that would only happen if cfg.save_intermediate_labels.
|
|
197
|
+
if cfg.save_intermediate_labels:
|
|
198
|
+
cur_labels_npy = store_dir / f"refined{cur_step}_labels.npy"
|
|
199
|
+
if cur_labels_npy.exists():
|
|
200
|
+
labels = np.load(cur_labels_npy)
|
|
201
|
+
sorting = DARTsortSorting.from_peeling_hdf5(cur_h5)
|
|
202
|
+
sorting = sorting.ephemeral_replace(labels=labels)
|
|
203
|
+
logger.info(
|
|
204
|
+
f"Resuming at step {cur_step + 1} with previous sorting from "
|
|
205
|
+
f"{cur_h5.name} and {cur_labels_npy.name}{mstr}."
|
|
206
|
+
)
|
|
207
|
+
return cur_step + 1, sorting, motion_est
|
|
208
|
+
|
|
209
|
+
# at this point, either the last round of peeling finished, or it
|
|
210
|
+
# didn't. we need to run peeler_is_done to check. but that's something
|
|
211
|
+
# done internally in each step anyway. so, what we do now is get the
|
|
212
|
+
# sorting for the PREVIOUS step which is the one we need to resume at
|
|
213
|
+
# the CURRENT step aka cur_step.
|
|
214
|
+
if cur_step == 0:
|
|
215
|
+
return cur_step, None, motion_est
|
|
216
|
+
|
|
217
|
+
prev_labels_npy = store_dir / f"refined{cur_step - 1}_labels.npy"
|
|
218
|
+
prev_labels = None
|
|
219
|
+
if prev_labels_npy.exists():
|
|
220
|
+
prev_labels = np.load(prev_labels_npy)
|
|
221
|
+
if cur_step == 1:
|
|
222
|
+
prev_h5 = sub_h5
|
|
223
|
+
else:
|
|
224
|
+
prev_h5 = store_dir / f"matching{cur_step - 1}.h5"
|
|
225
|
+
|
|
226
|
+
logger.info(
|
|
227
|
+
f"Resuming at step {cur_step} with previous sorting from "
|
|
228
|
+
f"{prev_h5.name} and {prev_labels_npy.name}{mstr}."
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
prev_sorting = DARTsortSorting.from_peeling_hdf5(prev_h5)
|
|
232
|
+
if prev_labels is not None:
|
|
233
|
+
prev_sorting = prev_sorting.ephemeral_replace(labels=prev_labels)
|
|
234
|
+
|
|
235
|
+
return cur_step, prev_sorting, motion_est
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
from typing import Callable, Optional, Tuple, Union
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
from jaxtyping import Float
|
|
5
|
+
from linear_operator.operators import to_dense
|
|
6
|
+
from linear_operator.operators._linear_operator import LinearOperator
|
|
7
|
+
from linear_operator.operators.block_linear_operator import BlockLinearOperator
|
|
8
|
+
from linear_operator.operators.diag_linear_operator import ConstantDiagLinearOperator
|
|
9
|
+
from linear_operator.operators.low_rank_root_linear_operator import (
|
|
10
|
+
LowRankRootLinearOperator,
|
|
11
|
+
)
|
|
12
|
+
from linear_operator.operators.sum_batch_linear_operator import SumBatchLinearOperator
|
|
13
|
+
from linear_operator.operators.sum_linear_operator import SumLinearOperator
|
|
14
|
+
from linear_operator.utils.cholesky import psd_safe_cholesky
|
|
15
|
+
from linear_operator.utils.memoize import cached
|
|
16
|
+
from torch import Tensor
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class LowRankRootSumLinearOperator(SumLinearOperator):
|
|
20
|
+
def __init__(self, *linear_ops, preconditioner_override=None):
|
|
21
|
+
if len(linear_ops) > 2:
|
|
22
|
+
raise RuntimeError(
|
|
23
|
+
"A LowRankRootSumLinearOperator can only have two components"
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
if sum(isinstance(op, LowRankRootLinearOperator) for op in linear_ops) > 1:
|
|
27
|
+
raise RuntimeError(
|
|
28
|
+
"A LowRankRootSumLinearOperator can have at most one LowRankRootLinearOperator base."
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
# keep track of my components
|
|
32
|
+
is_first = isinstance(linear_ops[0], LowRankRootLinearOperator)
|
|
33
|
+
self._root_op = linear_ops[1 - is_first]
|
|
34
|
+
self._other_op = linear_ops[is_first]
|
|
35
|
+
super().__init__(*linear_ops, preconditioner_override=preconditioner_override)
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
@cached(name="chol_cap_mat")
|
|
39
|
+
def chol_cap_mat(self):
|
|
40
|
+
V = self._root_op.root.mT
|
|
41
|
+
AinvU = self.AinvU
|
|
42
|
+
|
|
43
|
+
C = ConstantDiagLinearOperator(
|
|
44
|
+
torch.ones(*V.batch_shape, 1, device=V.device, dtype=V.dtype), V.shape[-2]
|
|
45
|
+
)
|
|
46
|
+
cap_mat = to_dense(C + V.matmul(AinvU))
|
|
47
|
+
chol_cap_mat = psd_safe_cholesky(cap_mat)
|
|
48
|
+
|
|
49
|
+
return chol_cap_mat
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
@cached(name="capinv_V")
|
|
53
|
+
def capinv_V(self):
|
|
54
|
+
V = self._root_op.root.mT.to_dense()
|
|
55
|
+
chol_cap_mat = self.chol_cap_mat
|
|
56
|
+
capinv_V = torch.cholesky_solve(V, chol_cap_mat)
|
|
57
|
+
return capinv_V
|
|
58
|
+
|
|
59
|
+
def _mul_constant(
|
|
60
|
+
self: Float[LinearOperator, "*batch M N"], other: Union[float, torch.Tensor]
|
|
61
|
+
) -> Float[LinearOperator, "*batch M N"]:
|
|
62
|
+
# We have to over-ride this here for the case where the constant is negative
|
|
63
|
+
if other > 0:
|
|
64
|
+
res = self.__class__(
|
|
65
|
+
self._root_op._mul_constant(other),
|
|
66
|
+
self._other_op._mul_constant(other),
|
|
67
|
+
)
|
|
68
|
+
else:
|
|
69
|
+
res = SumLinearOperator(
|
|
70
|
+
self._root_op._mul_constant(other),
|
|
71
|
+
self._other_op._mul_constant(other),
|
|
72
|
+
)
|
|
73
|
+
return res
|
|
74
|
+
|
|
75
|
+
def _preconditioner(
|
|
76
|
+
self,
|
|
77
|
+
) -> Tuple[Optional[Callable], Optional[LinearOperator], Optional[torch.Tensor]]:
|
|
78
|
+
return None, None, None
|
|
79
|
+
|
|
80
|
+
def _solve(
|
|
81
|
+
self: Float[LinearOperator, "... N N"],
|
|
82
|
+
rhs: Float[torch.Tensor, "... N C"],
|
|
83
|
+
preconditioner: Optional[
|
|
84
|
+
Callable[[Float[torch.Tensor, "... N C"]], Float[torch.Tensor, "... N C"]]
|
|
85
|
+
] = None,
|
|
86
|
+
num_tridiag: Optional[int] = 0,
|
|
87
|
+
Ainv_rhs=None,
|
|
88
|
+
) -> Union[
|
|
89
|
+
Float[torch.Tensor, "... N C"],
|
|
90
|
+
Tuple[
|
|
91
|
+
Float[torch.Tensor, "... N C"],
|
|
92
|
+
Float[
|
|
93
|
+
torch.Tensor, "..."
|
|
94
|
+
], # Note that in case of a tuple the second term size depends on num_tridiag
|
|
95
|
+
],
|
|
96
|
+
]:
|
|
97
|
+
A = self._other_op
|
|
98
|
+
AinvU = self.AinvU
|
|
99
|
+
V = self._root_op.root.mT
|
|
100
|
+
chol_cap_mat = self.chol_cap_mat
|
|
101
|
+
|
|
102
|
+
if Ainv_rhs is None:
|
|
103
|
+
Ainv_rhs = A.solve(rhs)
|
|
104
|
+
res = V.matmul(Ainv_rhs)
|
|
105
|
+
res = torch.cholesky_solve(res, chol_cap_mat)
|
|
106
|
+
# res = self.capinv_V.matmul(Ainv_rhs)
|
|
107
|
+
solve = Ainv_rhs.sub_(AinvU.matmul(res))
|
|
108
|
+
|
|
109
|
+
return solve
|
|
110
|
+
|
|
111
|
+
def _solve_preconditioner(self) -> Optional[Callable]:
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
def _sum_batch(self, dim: int) -> LinearOperator:
|
|
115
|
+
return SumBatchLinearOperator(self, dim)
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
@cached(name="AinvU")
|
|
119
|
+
def AinvU(self):
|
|
120
|
+
A = self._other_op
|
|
121
|
+
U = self._root_op.root
|
|
122
|
+
AinvU = A.solve(U.to_dense())
|
|
123
|
+
return AinvU
|
|
124
|
+
|
|
125
|
+
@cached(name="_logdet")
|
|
126
|
+
def _logdet(self):
|
|
127
|
+
chol_cap_mat = self.chol_cap_mat
|
|
128
|
+
logdet_cap_mat = 2 * torch.diagonal(
|
|
129
|
+
chol_cap_mat, offset=0, dim1=-2, dim2=-1
|
|
130
|
+
).log().sum(-1)
|
|
131
|
+
|
|
132
|
+
logdet_A = self._other_op.logdet()
|
|
133
|
+
logdet_term = logdet_cap_mat + logdet_A
|
|
134
|
+
|
|
135
|
+
return logdet_term
|
|
136
|
+
|
|
137
|
+
def __add__(
|
|
138
|
+
self: Float[LinearOperator, "... #M #N"],
|
|
139
|
+
other: Union[
|
|
140
|
+
Float[Tensor, "... #M #N"], Float[LinearOperator, "... #M #N"], float
|
|
141
|
+
],
|
|
142
|
+
) -> Union[Float[LinearOperator, "... M N"], Float[Tensor, "... M N"]]:
|
|
143
|
+
return self.__class__(self._root_op, self._other_op + other)
|
|
144
|
+
|
|
145
|
+
def inv_quad_logdet(
|
|
146
|
+
self: Float[LinearOperator, "*batch N N"],
|
|
147
|
+
inv_quad_rhs: Optional[
|
|
148
|
+
Union[Float[Tensor, "*batch N M"], Float[Tensor, "*batch N"]]
|
|
149
|
+
] = None,
|
|
150
|
+
logdet: Optional[bool] = False,
|
|
151
|
+
reduce_inv_quad: Optional[bool] = True,
|
|
152
|
+
) -> Tuple[
|
|
153
|
+
Optional[
|
|
154
|
+
Union[
|
|
155
|
+
Float[Tensor, "*batch M"], Float[Tensor, " *batch"], Float[Tensor, " 0"]
|
|
156
|
+
]
|
|
157
|
+
],
|
|
158
|
+
Optional[Float[Tensor, "..."]],
|
|
159
|
+
]:
|
|
160
|
+
if not self.is_square:
|
|
161
|
+
raise RuntimeError(
|
|
162
|
+
"inv_quad_logdet only operates on (batches of) square (positive semi-definite) LinearOperators. "
|
|
163
|
+
"Got a {} of size {}.".format(self.__class__.__name__, self.size())
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
if inv_quad_rhs is not None:
|
|
167
|
+
if self.dim() == 2 and inv_quad_rhs.dim() == 1:
|
|
168
|
+
if self.shape[-1] != inv_quad_rhs.numel():
|
|
169
|
+
raise RuntimeError(
|
|
170
|
+
"LinearOperator (size={}) cannot be multiplied with right-hand-side Tensor (size={}).".format(
|
|
171
|
+
self.shape, inv_quad_rhs.shape
|
|
172
|
+
)
|
|
173
|
+
)
|
|
174
|
+
elif self.dim() != inv_quad_rhs.dim():
|
|
175
|
+
raise RuntimeError(
|
|
176
|
+
"LinearOperator (size={}) and right-hand-side Tensor (size={}) should have the same number "
|
|
177
|
+
"of dimensions.".format(self.shape, inv_quad_rhs.shape)
|
|
178
|
+
)
|
|
179
|
+
elif (
|
|
180
|
+
self.batch_shape != inv_quad_rhs.shape[:-2]
|
|
181
|
+
or self.shape[-1] != inv_quad_rhs.shape[-2]
|
|
182
|
+
):
|
|
183
|
+
raise RuntimeError(
|
|
184
|
+
"LinearOperator (size={}) cannot be multiplied with right-hand-side Tensor (size={}).".format(
|
|
185
|
+
self.shape, inv_quad_rhs.shape
|
|
186
|
+
)
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
inv_quad_term, logdet_term = None, None
|
|
190
|
+
|
|
191
|
+
if inv_quad_rhs is not None:
|
|
192
|
+
self_inv_rhs = self._solve(inv_quad_rhs)
|
|
193
|
+
inv_quad_term = (inv_quad_rhs * self_inv_rhs).sum(dim=-2)
|
|
194
|
+
if reduce_inv_quad:
|
|
195
|
+
inv_quad_term = inv_quad_term.sum(dim=-1)
|
|
196
|
+
|
|
197
|
+
if logdet:
|
|
198
|
+
logdet_term = self._logdet()
|
|
199
|
+
|
|
200
|
+
return inv_quad_term, logdet_term
|
|
201
|
+
|
|
202
|
+
def solve(
|
|
203
|
+
self: Float[LinearOperator, "... N N"],
|
|
204
|
+
right_tensor: Union[Float[Tensor, "... N P"], Float[Tensor, " N"]],
|
|
205
|
+
left_tensor: Optional[Float[Tensor, "... O N"]] = None,
|
|
206
|
+
) -> Union[
|
|
207
|
+
Float[Tensor, "... N P"],
|
|
208
|
+
Float[Tensor, "... N"],
|
|
209
|
+
Float[Tensor, "... O P"],
|
|
210
|
+
Float[Tensor, "... O"],
|
|
211
|
+
]:
|
|
212
|
+
if not self.is_square:
|
|
213
|
+
raise RuntimeError(
|
|
214
|
+
"solve only operates on (batches of) square (positive semi-definite) LinearOperators. "
|
|
215
|
+
"Got a {} of size {}.".format(self.__class__.__name__, self.size())
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
if self.dim() == 2 and right_tensor.dim() == 1:
|
|
219
|
+
if self.shape[-1] != right_tensor.numel():
|
|
220
|
+
raise RuntimeError(
|
|
221
|
+
"LinearOperator (size={}) cannot be multiplied with right-hand-side Tensor (size={}).".format(
|
|
222
|
+
self.shape, right_tensor.shape
|
|
223
|
+
)
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
squeeze_solve = False
|
|
227
|
+
if right_tensor.ndimension() == 1:
|
|
228
|
+
right_tensor = right_tensor.unsqueeze(-1)
|
|
229
|
+
squeeze_solve = True
|
|
230
|
+
|
|
231
|
+
solve = self._solve(right_tensor)
|
|
232
|
+
if squeeze_solve:
|
|
233
|
+
solve = solve.squeeze(-1)
|
|
234
|
+
|
|
235
|
+
if left_tensor is not None:
|
|
236
|
+
return left_tensor @ solve
|
|
237
|
+
else:
|
|
238
|
+
return solve
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class NonSquareBlockLinearOperator(BlockLinearOperator):
|
|
242
|
+
@property
|
|
243
|
+
def num_blocks(self) -> int:
|
|
244
|
+
return self.base_linear_op.size(-3)
|
|
245
|
+
|
|
246
|
+
def _size(self):
|
|
247
|
+
shape = list(self.base_linear_op.shape)
|
|
248
|
+
shape[-2] *= shape[-3]
|
|
249
|
+
shape[-1] *= shape[-3]
|
|
250
|
+
del shape[-3]
|
|
251
|
+
return torch.Size(shape)
|
|
252
|
+
|
|
253
|
+
def _add_batch_dim(
|
|
254
|
+
self: Float[LinearOperator, "*batch1 M P"],
|
|
255
|
+
other: Float[torch.Tensor, "*batch2 N C"],
|
|
256
|
+
) -> Float[torch.Tensor, "batch2 ... C"]:
|
|
257
|
+
*batch_shape, num_rows, num_cols = other.shape
|
|
258
|
+
batch_shape = list(batch_shape)
|
|
259
|
+
|
|
260
|
+
batch_shape.append(self.num_blocks)
|
|
261
|
+
other = other.view(*batch_shape, num_rows // self.num_blocks, num_cols)
|
|
262
|
+
return other
|
|
263
|
+
|
|
264
|
+
def _remove_batch_dim(self, other):
|
|
265
|
+
shape = list(other.shape)
|
|
266
|
+
del shape[-3]
|
|
267
|
+
shape[-2] *= self.num_blocks
|
|
268
|
+
other = other.reshape(*shape)
|
|
269
|
+
return other
|