DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
from concurrent.futures import CancelledError, ProcessPoolExecutor
|
|
2
|
+
from concurrent.futures import ThreadPoolExecutor as _ThreadPoolExecutor
|
|
3
|
+
import multiprocessing
|
|
4
|
+
import os
|
|
5
|
+
from multiprocessing import get_context
|
|
6
|
+
import queue
|
|
7
|
+
|
|
8
|
+
from . import job_util
|
|
9
|
+
|
|
10
|
+
have_cloudpickle = False
|
|
11
|
+
cloudpickle = None
|
|
12
|
+
try:
|
|
13
|
+
import cloudpickle # type: ignore[reportMissingImports]
|
|
14
|
+
|
|
15
|
+
have_cloudpickle = True
|
|
16
|
+
except ImportError:
|
|
17
|
+
try:
|
|
18
|
+
from joblib.externals import cloudpickle
|
|
19
|
+
|
|
20
|
+
have_cloudpickle = True
|
|
21
|
+
except ImportError:
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ThreadPoolExecutor(_ThreadPoolExecutor):
|
|
26
|
+
"""shim for same api ([mp_]context args)"""
|
|
27
|
+
|
|
28
|
+
def __init__(
|
|
29
|
+
self,
|
|
30
|
+
max_workers=None,
|
|
31
|
+
mp_context=None,
|
|
32
|
+
initializer=None,
|
|
33
|
+
initargs=(),
|
|
34
|
+
context=None,
|
|
35
|
+
):
|
|
36
|
+
super().__init__(
|
|
37
|
+
max_workers=max_workers, initializer=initializer, initargs=initargs
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class MockFuture:
|
|
42
|
+
"""A non-concurrent class for mocking the concurrent.futures API."""
|
|
43
|
+
|
|
44
|
+
def __init__(self, f, *args):
|
|
45
|
+
self.f = f
|
|
46
|
+
self.args = args
|
|
47
|
+
|
|
48
|
+
def result(self):
|
|
49
|
+
return self.f(*self.args)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class MockPoolExecutor:
|
|
53
|
+
"""A non-concurrent class for mocking the concurrent.futures API."""
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
max_workers=None,
|
|
58
|
+
mp_context=None,
|
|
59
|
+
initializer=None,
|
|
60
|
+
initargs=(),
|
|
61
|
+
context=None,
|
|
62
|
+
):
|
|
63
|
+
self.initargs = initargs
|
|
64
|
+
self.initializer = initializer
|
|
65
|
+
self.initialized = False
|
|
66
|
+
self.cancelled = False
|
|
67
|
+
|
|
68
|
+
def _initialize(self):
|
|
69
|
+
if self.initialized:
|
|
70
|
+
return
|
|
71
|
+
if self.initializer is not None:
|
|
72
|
+
self.initializer(*self.initargs)
|
|
73
|
+
self.initialized = True
|
|
74
|
+
|
|
75
|
+
def map(self, function, *iterables, timeout=None, chunksize=1):
|
|
76
|
+
self._initialize()
|
|
77
|
+
for result in map(function, *iterables):
|
|
78
|
+
yield result
|
|
79
|
+
if self.cancelled:
|
|
80
|
+
raise CancelledError()
|
|
81
|
+
|
|
82
|
+
def __enter__(self):
|
|
83
|
+
return self
|
|
84
|
+
|
|
85
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
86
|
+
return
|
|
87
|
+
|
|
88
|
+
def submit(self, f, *args):
|
|
89
|
+
self._initialize()
|
|
90
|
+
return MockFuture(f, *args)
|
|
91
|
+
|
|
92
|
+
def shutdown(self, wait=True, *, cancel_futures=False):
|
|
93
|
+
if cancel_futures:
|
|
94
|
+
self.cancelled = True
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class MockQueue:
|
|
98
|
+
"""Another helper class for turning off concurrency when debugging."""
|
|
99
|
+
|
|
100
|
+
def __init__(self):
|
|
101
|
+
self.q = []
|
|
102
|
+
self.put = self.q.append
|
|
103
|
+
self.get = lambda: self.q.pop(0)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def cloudpickle_run(fn, args):
|
|
107
|
+
assert cloudpickle is not None
|
|
108
|
+
fn = cloudpickle.loads(fn)
|
|
109
|
+
args, kwargs = cloudpickle.loads(args)
|
|
110
|
+
# return cloudpickle.dumps(fn(*args, **kwargs))
|
|
111
|
+
return fn(*args, **kwargs)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# def uncloudpickle(future):
|
|
115
|
+
# res = future.result()
|
|
116
|
+
# future._result = cloudpickle.loads(res)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class CloudpicklePoolExecutor(ProcessPoolExecutor):
|
|
120
|
+
def submit(self, fn, /, *args, **kwargs):
|
|
121
|
+
assert cloudpickle is not None
|
|
122
|
+
args = cloudpickle.dumps((args, kwargs))
|
|
123
|
+
future = super().submit(cloudpickle_run, cloudpickle.dumps(fn), args)
|
|
124
|
+
# future.add_done_callback(uncloudpickle_callback)
|
|
125
|
+
return future
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def rank_init(queue):
|
|
129
|
+
rank_init.rank = queue.get() # type: ignore
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def pool_from_cfg(computation_config=None, with_rank_queue=False, check_local=False):
|
|
133
|
+
if computation_config is None:
|
|
134
|
+
computation_config = job_util.get_global_computation_config()
|
|
135
|
+
|
|
136
|
+
return get_pool(
|
|
137
|
+
computation_config.actual_n_jobs(),
|
|
138
|
+
cls=computation_config.executor,
|
|
139
|
+
with_rank_queue=with_rank_queue,
|
|
140
|
+
check_local=check_local,
|
|
141
|
+
multi_gpu=computation_config.is_multi_gpu(),
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def get_pool(
|
|
146
|
+
n_jobs,
|
|
147
|
+
context="spawn",
|
|
148
|
+
cls="threading_unless_multigpu",
|
|
149
|
+
with_rank_queue=False,
|
|
150
|
+
rank_queue_empty=False,
|
|
151
|
+
n_tasks=None,
|
|
152
|
+
max_tasks_per_child=None,
|
|
153
|
+
check_local=False,
|
|
154
|
+
multi_gpu=False,
|
|
155
|
+
):
|
|
156
|
+
if n_jobs == -1:
|
|
157
|
+
try:
|
|
158
|
+
n_jobs = os.process_cpu_count() # type: ignore
|
|
159
|
+
except AttributeError:
|
|
160
|
+
n_jobs = multiprocessing.cpu_count()
|
|
161
|
+
do_parallel = n_jobs >= 1
|
|
162
|
+
n_jobs = max(1, n_jobs)
|
|
163
|
+
|
|
164
|
+
if isinstance(cls, str):
|
|
165
|
+
if cls == "threading_unless_multigpu":
|
|
166
|
+
if (n_jobs > 1 and multi_gpu) or os.name == "nt":
|
|
167
|
+
cls = "ProcessPoolExecutor"
|
|
168
|
+
else:
|
|
169
|
+
cls = "ThreadPoolExecutor"
|
|
170
|
+
if cls == "CloudpicklePoolExecutor":
|
|
171
|
+
cls = CloudpicklePoolExecutor
|
|
172
|
+
elif cls == "ThreadPoolExecutor":
|
|
173
|
+
cls = ThreadPoolExecutor
|
|
174
|
+
elif cls == "ProcessPoolExecutor":
|
|
175
|
+
cls = ProcessPoolExecutor
|
|
176
|
+
elif cls == "MockPoolExecutor":
|
|
177
|
+
cls = MockPoolExecutor
|
|
178
|
+
else:
|
|
179
|
+
assert False
|
|
180
|
+
|
|
181
|
+
if cls == CloudpicklePoolExecutor and not have_cloudpickle:
|
|
182
|
+
cls = ProcessPoolExecutor
|
|
183
|
+
|
|
184
|
+
Executor = cls if do_parallel else MockPoolExecutor
|
|
185
|
+
is_local = cls in (MockPoolExecutor, ThreadPoolExecutor)
|
|
186
|
+
context = get_context(context)
|
|
187
|
+
|
|
188
|
+
if with_rank_queue:
|
|
189
|
+
if do_parallel and is_local:
|
|
190
|
+
rank_queue = queue.SimpleQueue()
|
|
191
|
+
elif do_parallel:
|
|
192
|
+
manager = context.Manager()
|
|
193
|
+
rank_queue = manager.Queue()
|
|
194
|
+
else:
|
|
195
|
+
rank_queue = MockQueue()
|
|
196
|
+
|
|
197
|
+
if not rank_queue_empty:
|
|
198
|
+
n_repeats = 1
|
|
199
|
+
if max_tasks_per_child is not None:
|
|
200
|
+
n_repeats = n_tasks // max_tasks_per_child + 1
|
|
201
|
+
for _ in range(n_repeats):
|
|
202
|
+
for rank in range(n_jobs):
|
|
203
|
+
rank_queue.put(rank)
|
|
204
|
+
|
|
205
|
+
if check_local:
|
|
206
|
+
return n_jobs, Executor, context, rank_queue, is_local
|
|
207
|
+
return n_jobs, Executor, context, rank_queue
|
|
208
|
+
|
|
209
|
+
if check_local:
|
|
210
|
+
return n_jobs, Executor, context, is_local
|
|
211
|
+
return n_jobs, Executor, context
|
dartsort/util/nn_util.py
ADDED
|
@@ -0,0 +1,461 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
import torch.nn.functional as F
|
|
3
|
+
from torch import nn
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class AttentionBlock(nn.Module):
|
|
7
|
+
def __init__(self, embed_dim, num_heads=1, dropout=0.0):
|
|
8
|
+
super().__init__()
|
|
9
|
+
self.embed_dim = embed_dim
|
|
10
|
+
self.num_heads = num_heads
|
|
11
|
+
self.scaling = embed_dim**-0.5
|
|
12
|
+
|
|
13
|
+
self.q_proj = nn.Linear(embed_dim, embed_dim)
|
|
14
|
+
self.k_proj = nn.Linear(embed_dim, embed_dim)
|
|
15
|
+
self.v_proj = nn.Linear(embed_dim, embed_dim)
|
|
16
|
+
self.out_proj = nn.Linear(embed_dim, embed_dim)
|
|
17
|
+
self.dropout = nn.Dropout(dropout)
|
|
18
|
+
|
|
19
|
+
def forward(self, x):
|
|
20
|
+
# x: [batch_size, seq_length, embed_dim]
|
|
21
|
+
q = self.q_proj(x) # [batch_size, seq_length, embed_dim]
|
|
22
|
+
k = self.k_proj(x) # [batch_size, seq_length, embed_dim]
|
|
23
|
+
v = self.v_proj(x) # [batch_size, seq_length, embed_dim]
|
|
24
|
+
|
|
25
|
+
# Compute attention scores
|
|
26
|
+
attn_scores = torch.matmul(q, k.transpose(-2, -1)) * self.scaling
|
|
27
|
+
# [batch_size, seq_length, seq_length]
|
|
28
|
+
attn_weights = torch.softmax(attn_scores, dim=-1)
|
|
29
|
+
attn_weights = self.dropout(attn_weights)
|
|
30
|
+
|
|
31
|
+
# Apply attention weights
|
|
32
|
+
# [batch_size, seq_length, embed_dim]
|
|
33
|
+
attn_output = torch.matmul(attn_weights, v)
|
|
34
|
+
return self.out_proj(attn_output)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ResidualBlock(nn.Module):
|
|
38
|
+
def __init__(self, input_dim, output_dim, norm_kind="batchnorm"):
|
|
39
|
+
super().__init__()
|
|
40
|
+
self.input_dim = input_dim
|
|
41
|
+
self.output_dim = output_dim
|
|
42
|
+
self.norm_kind = norm_kind
|
|
43
|
+
|
|
44
|
+
self.linear = nn.Linear(input_dim, output_dim)
|
|
45
|
+
self.norm = get_norm(output_dim, norm_kind)
|
|
46
|
+
self.relu = nn.ReLU()
|
|
47
|
+
self.projection = None
|
|
48
|
+
if input_dim != output_dim:
|
|
49
|
+
self.projection = nn.Linear(input_dim, output_dim)
|
|
50
|
+
|
|
51
|
+
def forward(self, x):
|
|
52
|
+
residual = x
|
|
53
|
+
out = self.linear(x)
|
|
54
|
+
if self.norm is not None:
|
|
55
|
+
out = self.norm(out)
|
|
56
|
+
out = self.relu(out)
|
|
57
|
+
if self.projection is not None:
|
|
58
|
+
residual = self.projection(residual)
|
|
59
|
+
return out + residual
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ConcatResidualBlock(nn.Module):
|
|
63
|
+
def __init__(self, input_dim, output_dim, norm_kind="batchnorm"):
|
|
64
|
+
super().__init__()
|
|
65
|
+
self.input_dim = input_dim
|
|
66
|
+
self.output_dim = output_dim
|
|
67
|
+
self.norm_kind = norm_kind
|
|
68
|
+
|
|
69
|
+
self.linear = nn.Linear(input_dim, output_dim)
|
|
70
|
+
self.project_back = nn.Linear(input_dim + output_dim, output_dim)
|
|
71
|
+
self.norm = get_norm(output_dim, norm_kind)
|
|
72
|
+
|
|
73
|
+
self.relu = nn.ReLU()
|
|
74
|
+
|
|
75
|
+
def forward(self, x):
|
|
76
|
+
out = self.linear(x)
|
|
77
|
+
if self.norm is not None:
|
|
78
|
+
out = self.norm(out)
|
|
79
|
+
out = self.relu(out)
|
|
80
|
+
|
|
81
|
+
concatenated = torch.cat([x, out], dim=-1)
|
|
82
|
+
projected = self.project_back(concatenated)
|
|
83
|
+
return projected
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class LinearResidualBlock(nn.Module):
|
|
87
|
+
def __init__(self, input_dim, hidden_dim, norm_kind=None):
|
|
88
|
+
super().__init__()
|
|
89
|
+
self.input_dim = input_dim
|
|
90
|
+
self.hidden_dim = hidden_dim
|
|
91
|
+
self.norm_kind = norm_kind
|
|
92
|
+
|
|
93
|
+
self.linear1 = nn.Linear(input_dim, hidden_dim)
|
|
94
|
+
self.linear2 = nn.Linear(hidden_dim, input_dim)
|
|
95
|
+
self.norm = get_norm(hidden_dim, norm_kind)
|
|
96
|
+
|
|
97
|
+
self.relu = nn.ReLU()
|
|
98
|
+
|
|
99
|
+
def forward(self, x):
|
|
100
|
+
out = self.linear1(x)
|
|
101
|
+
if self.norm is not None:
|
|
102
|
+
out = self.norm(out)
|
|
103
|
+
out = self.relu(out)
|
|
104
|
+
out = self.linear2(out)
|
|
105
|
+
return out + x
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class UNetLinear(nn.Module):
|
|
109
|
+
def __init__(
|
|
110
|
+
self,
|
|
111
|
+
input_dim,
|
|
112
|
+
half_hidden_dims,
|
|
113
|
+
nonlinearity,
|
|
114
|
+
norm_kind=None,
|
|
115
|
+
):
|
|
116
|
+
self.blocks_in = nn.ModuleList()
|
|
117
|
+
self.blocks_out = nn.ModuleList()
|
|
118
|
+
|
|
119
|
+
inner_gated = nonlinearity = torch.nn.GLU
|
|
120
|
+
|
|
121
|
+
*in_dims, middle_dim = half_hidden_dims
|
|
122
|
+
out_dims = in_dims[::-1]
|
|
123
|
+
|
|
124
|
+
current_dim = input_dim
|
|
125
|
+
out_dims_in = []
|
|
126
|
+
for out_dim in half_hiddden_dims:
|
|
127
|
+
block_layers = []
|
|
128
|
+
block_layers.append(nn.Linear(current_dim, (1 + inner_gated) * out_dim))
|
|
129
|
+
norm = get_norm((1 + inner_gated) * out_dim, norm_kind)
|
|
130
|
+
if norm is not None:
|
|
131
|
+
block_layers.append(norm)
|
|
132
|
+
block_layers.append(nonlinearity())
|
|
133
|
+
self.blocks_in.append(nn.Sequential(*block_layers))
|
|
134
|
+
current_dim = out_dim
|
|
135
|
+
out_dims_in.append(out_dim)
|
|
136
|
+
|
|
137
|
+
block_layers = []
|
|
138
|
+
block_layers.append(nn.Linear(current_dim, (1 + inner_gated) * middle_dim))
|
|
139
|
+
norm = get_norm((1 + inner_gated) * middle_dim, norm_kind)
|
|
140
|
+
if norm is not None:
|
|
141
|
+
block_layers.append(norm)
|
|
142
|
+
block_layers.append(nonlinearity())
|
|
143
|
+
self.middle_block = nn.Sequential(*block_layers)
|
|
144
|
+
current_dim = middle_dim
|
|
145
|
+
|
|
146
|
+
for out_dim in out_dims:
|
|
147
|
+
cat_dim = out_dims_in.pop()
|
|
148
|
+
current_dim = current_dim + cat_dim
|
|
149
|
+
|
|
150
|
+
block_layers = [Cat()]
|
|
151
|
+
block_layers.append(nn.Linear(current_dim, (1 + inner_gated) * out_dim))
|
|
152
|
+
norm = get_norm((1 + inner_gated) * out_dim, norm_kind)
|
|
153
|
+
if norm is not None:
|
|
154
|
+
block_layers.append(norm)
|
|
155
|
+
block_layers.append(nonlinearity())
|
|
156
|
+
self.blocks_out.append(nn.Sequential(*block_layers))
|
|
157
|
+
current_dim = out_dim
|
|
158
|
+
self.final_dim = current_dim
|
|
159
|
+
assert not out_dims_in
|
|
160
|
+
|
|
161
|
+
def forward(self, x):
|
|
162
|
+
stack = []
|
|
163
|
+
for b in self.blocks_in:
|
|
164
|
+
x = b(x)
|
|
165
|
+
stack.append(x)
|
|
166
|
+
x = self.middle_block(x)
|
|
167
|
+
for b in self.blocks_out:
|
|
168
|
+
x = b((x, stack.pop()))
|
|
169
|
+
return x
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def get_mlp(
|
|
173
|
+
input_dim,
|
|
174
|
+
hidden_dims,
|
|
175
|
+
output_dim,
|
|
176
|
+
norm_kind="batchnorm",
|
|
177
|
+
res_type="none",
|
|
178
|
+
output_layer="linear",
|
|
179
|
+
nonlinearity="ReLU",
|
|
180
|
+
attention_layer=False,
|
|
181
|
+
num_heads=4,
|
|
182
|
+
):
|
|
183
|
+
layers = []
|
|
184
|
+
if isinstance(nonlinearity, str):
|
|
185
|
+
nonlinearity = getattr(torch.nn, nonlinearity)
|
|
186
|
+
gated = nonlinearity == torch.nn.GLU
|
|
187
|
+
|
|
188
|
+
if res_type == "blocks_concat":
|
|
189
|
+
current_dim = input_dim
|
|
190
|
+
for out_dim in hidden_dims:
|
|
191
|
+
layers.append(ConcatResidualBlock(current_dim, out_dim, norm_kind))
|
|
192
|
+
current_dim = out_dim
|
|
193
|
+
if attention_layer:
|
|
194
|
+
layers.append(AttentionBlock(current_dim, num_heads=num_heads))
|
|
195
|
+
final_dim = current_dim
|
|
196
|
+
|
|
197
|
+
elif res_type == "blocks_add":
|
|
198
|
+
input_dims = [input_dim, *hidden_dims[:-1]]
|
|
199
|
+
for ind, outd in zip(input_dims, hidden_dims):
|
|
200
|
+
layers.append(ResidualBlock(ind, outd, norm_kind))
|
|
201
|
+
if attention_layer:
|
|
202
|
+
layers.append(AttentionBlock(outd, num_heads=num_heads))
|
|
203
|
+
final_dim = hidden_dims[-1] if hidden_dims else input_dim
|
|
204
|
+
|
|
205
|
+
elif res_type == "blocks_linear":
|
|
206
|
+
assert len(hidden_dims) > 1
|
|
207
|
+
compression_dim = hidden_dims[0]
|
|
208
|
+
layers.append(nn.Linear(input_dim, compression_dim))
|
|
209
|
+
for hd in hidden_dims[1:]:
|
|
210
|
+
layers.append(LinearResidualBlock(compression_dim, hd, norm_kind=norm_kind))
|
|
211
|
+
final_dim = compression_dim
|
|
212
|
+
|
|
213
|
+
elif res_type == "unet_linear":
|
|
214
|
+
layers.append(UNetLinear(input_dim, hidden_dims, out_dim, nonlinearity))
|
|
215
|
+
final_dim = layers[-1].final_dim
|
|
216
|
+
|
|
217
|
+
elif res_type in ("none", "outer"):
|
|
218
|
+
# res_type == "none" is handled in get_waveform_mlp
|
|
219
|
+
# res_type in get_mlp is the *inner* residual type in each block
|
|
220
|
+
current_dim = input_dim
|
|
221
|
+
for out_dim in hidden_dims:
|
|
222
|
+
layers.append(nn.Linear(current_dim, (1 + gated) * out_dim))
|
|
223
|
+
norm = get_norm((1 + gated) * out_dim, norm_kind)
|
|
224
|
+
if norm is not None:
|
|
225
|
+
layers.append(norm)
|
|
226
|
+
layers.append(nonlinearity())
|
|
227
|
+
current_dim = out_dim
|
|
228
|
+
if attention_layer:
|
|
229
|
+
layers.append(AttentionBlock(current_dim, num_heads=num_heads))
|
|
230
|
+
final_dim = current_dim
|
|
231
|
+
|
|
232
|
+
else:
|
|
233
|
+
raise ValueError(f"Unsupported res_type: {res_type}")
|
|
234
|
+
|
|
235
|
+
if output_layer == "linear":
|
|
236
|
+
layers.append(nn.Linear(final_dim, output_dim))
|
|
237
|
+
elif output_layer == "gated_linear":
|
|
238
|
+
layers.append(nn.Linear(final_dim, 2 * output_dim))
|
|
239
|
+
layers.append(nn.GLU())
|
|
240
|
+
else:
|
|
241
|
+
assert False
|
|
242
|
+
|
|
243
|
+
return nn.Sequential(*layers)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def get_waveform_mlp(
|
|
247
|
+
spike_length_samples,
|
|
248
|
+
n_input_channels,
|
|
249
|
+
hidden_dims,
|
|
250
|
+
output_dim,
|
|
251
|
+
input_includes_mask=True,
|
|
252
|
+
norm_kind="batchnorm",
|
|
253
|
+
scaling=None,
|
|
254
|
+
log_transform=False,
|
|
255
|
+
channelwise_dropout_p=0.0,
|
|
256
|
+
separated_mask_input=False,
|
|
257
|
+
initial_conv_fullheight=False,
|
|
258
|
+
final_conv_fullheight=False,
|
|
259
|
+
return_initial_shape=False,
|
|
260
|
+
res_type="none",
|
|
261
|
+
output_layer="linear",
|
|
262
|
+
nonlinearity="ReLU",
|
|
263
|
+
attention_layer=False,
|
|
264
|
+
num_heads=4,
|
|
265
|
+
):
|
|
266
|
+
input_dim = n_input_channels * (spike_length_samples + input_includes_mask)
|
|
267
|
+
layers = []
|
|
268
|
+
if log_transform:
|
|
269
|
+
layers.append(WaveformOnly(LogTransform()))
|
|
270
|
+
|
|
271
|
+
if initial_conv_fullheight:
|
|
272
|
+
# what Conv1d considers channels is actually time (conv1d is ncl).
|
|
273
|
+
# so this is matmul over time, and kernel size is 1 to be separate over chans
|
|
274
|
+
conv = nn.Conv1d(spike_length_samples, spike_length_samples, kernel_size=1)
|
|
275
|
+
layers.append(WaveformOnly(conv))
|
|
276
|
+
norm = get_norm(n_input_channels, norm_kind)
|
|
277
|
+
if norm is not None:
|
|
278
|
+
layers.append(
|
|
279
|
+
WaveformOnly(nn.Sequential(Permute(0, 2, 1), norm, Permute(0, 2, 1)))
|
|
280
|
+
)
|
|
281
|
+
layers.append(WaveformOnly(nn.ReLU()))
|
|
282
|
+
|
|
283
|
+
if separated_mask_input:
|
|
284
|
+
layers.append(Cat(dim=1))
|
|
285
|
+
|
|
286
|
+
if channelwise_dropout_p:
|
|
287
|
+
layers.append(ChannelwiseDropout(channelwise_dropout_p))
|
|
288
|
+
|
|
289
|
+
layers.append(nn.Flatten())
|
|
290
|
+
mlp = get_mlp(
|
|
291
|
+
input_dim,
|
|
292
|
+
hidden_dims,
|
|
293
|
+
output_dim,
|
|
294
|
+
norm_kind=norm_kind,
|
|
295
|
+
res_type=res_type,
|
|
296
|
+
nonlinearity=nonlinearity,
|
|
297
|
+
attention_layer=attention_layer,
|
|
298
|
+
output_layer=output_layer,
|
|
299
|
+
num_heads=num_heads,
|
|
300
|
+
)
|
|
301
|
+
layers.append(mlp)
|
|
302
|
+
|
|
303
|
+
if return_initial_shape:
|
|
304
|
+
layers.append(nn.Unflatten(-1, (spike_length_samples, n_input_channels)))
|
|
305
|
+
if final_conv_fullheight:
|
|
306
|
+
norm = get_norm(n_input_channels, norm_kind)
|
|
307
|
+
if norm is not None:
|
|
308
|
+
layers.append(nn.Sequential(Permute(0, 2, 1), norm, Permute(0, 2, 1)))
|
|
309
|
+
layers.append(nn.ReLU())
|
|
310
|
+
conv = nn.Conv1d(spike_length_samples, spike_length_samples, kernel_size=1)
|
|
311
|
+
layers.append(conv)
|
|
312
|
+
|
|
313
|
+
net = nn.Sequential(*layers)
|
|
314
|
+
|
|
315
|
+
if scaling not in (None, "none"):
|
|
316
|
+
net = Rescaling(net, scaling, waveform_only=True)
|
|
317
|
+
|
|
318
|
+
if res_type == "outer":
|
|
319
|
+
net = WaveformOnlyResidualForm(net)
|
|
320
|
+
|
|
321
|
+
if log_transform:
|
|
322
|
+
layers.append(WaveformOnly(ExpTransform()))
|
|
323
|
+
|
|
324
|
+
return net
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def get_norm(n_features, norm_kind=None):
|
|
328
|
+
if norm_kind == "batchnorm":
|
|
329
|
+
return nn.BatchNorm1d(n_features)
|
|
330
|
+
if norm_kind == "layernorm":
|
|
331
|
+
return nn.LayerNorm(n_features)
|
|
332
|
+
assert norm_kind in ("none", None)
|
|
333
|
+
return None
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
class ResidualForm(nn.Module):
|
|
337
|
+
|
|
338
|
+
def __init__(self, module):
|
|
339
|
+
super().__init__()
|
|
340
|
+
self.module = module
|
|
341
|
+
|
|
342
|
+
def forward(self, input):
|
|
343
|
+
output = self.module(input)
|
|
344
|
+
return input + output
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
class Rescaling(nn.Module):
|
|
348
|
+
def __init__(self, module, scale_kind="std", waveform_only=False):
|
|
349
|
+
super().__init__()
|
|
350
|
+
self.scale_kind = scale_kind
|
|
351
|
+
self.module = module
|
|
352
|
+
self.waveform_only = waveform_only
|
|
353
|
+
|
|
354
|
+
def forward(self, input):
|
|
355
|
+
if self.waveform_only:
|
|
356
|
+
input, masks = input
|
|
357
|
+
dim = tuple(range(1, input.ndim))
|
|
358
|
+
if self.scale_kind == "max":
|
|
359
|
+
scales = input.abs().amax(dim=dim, keepdim=True)
|
|
360
|
+
elif self.scale_kind == "std":
|
|
361
|
+
scales = input.std(dim=dim, keepdim=True)
|
|
362
|
+
else:
|
|
363
|
+
assert False
|
|
364
|
+
if self.waveform_only:
|
|
365
|
+
output = self.module((input / scales, masks))
|
|
366
|
+
else:
|
|
367
|
+
output = self.module(input / scales)
|
|
368
|
+
return output * scales
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
class LogTransform(nn.Module):
|
|
372
|
+
def forward(self, input):
|
|
373
|
+
sgn = torch.sign(input)
|
|
374
|
+
log = torch.log1p(input.abs())
|
|
375
|
+
return log * sgn
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
class ExpTransform(nn.Module):
|
|
379
|
+
def forward(self, input):
|
|
380
|
+
sgn = torch.sign(input)
|
|
381
|
+
exp = torch.expm1(input.abs())
|
|
382
|
+
return exp * sgn
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
class WaveformOnlyResidualForm(nn.Module):
|
|
386
|
+
def __init__(self, module):
|
|
387
|
+
super().__init__()
|
|
388
|
+
self.module = module
|
|
389
|
+
|
|
390
|
+
def forward(self, inputs):
|
|
391
|
+
waveforms, masks = inputs
|
|
392
|
+
output = self.module(inputs)
|
|
393
|
+
return waveforms + output
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
class ChannelwiseDropout(nn.Module):
|
|
397
|
+
def __init__(self, p):
|
|
398
|
+
super().__init__()
|
|
399
|
+
self.p = p
|
|
400
|
+
|
|
401
|
+
def forward(self, waveforms):
|
|
402
|
+
res = F.dropout1d(waveforms.permute(0, 2, 1), p=self.p, training=self.training)
|
|
403
|
+
res = res.permute(0, 2, 1)
|
|
404
|
+
return res
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
class Cat(nn.Module):
|
|
408
|
+
def __init__(self, dim=-1):
|
|
409
|
+
super().__init__()
|
|
410
|
+
self.dim = dim
|
|
411
|
+
|
|
412
|
+
def forward(self, inputs):
|
|
413
|
+
return torch.cat(inputs, dim=self.dim)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
class Permute(nn.Module):
|
|
417
|
+
def __init__(self, *dims):
|
|
418
|
+
super().__init__()
|
|
419
|
+
self.dims = dims
|
|
420
|
+
|
|
421
|
+
def forward(self, inputs):
|
|
422
|
+
return inputs.permute(*self.dims)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
class WaveformOnly(nn.Module):
|
|
426
|
+
def __init__(self, module):
|
|
427
|
+
super().__init__()
|
|
428
|
+
self.module = module
|
|
429
|
+
|
|
430
|
+
def forward(self, inputs):
|
|
431
|
+
waveforms, masks = inputs
|
|
432
|
+
return self.module(waveforms), masks
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
# is this what they want us to do??
|
|
436
|
+
if hasattr(torch.serialization, "add_safe_globals"):
|
|
437
|
+
torch.serialization.add_safe_globals(
|
|
438
|
+
[
|
|
439
|
+
ResidualForm,
|
|
440
|
+
WaveformOnlyResidualForm,
|
|
441
|
+
ChannelwiseDropout,
|
|
442
|
+
Cat,
|
|
443
|
+
Permute,
|
|
444
|
+
WaveformOnly,
|
|
445
|
+
Rescaling,
|
|
446
|
+
LogTransform,
|
|
447
|
+
ExpTransform,
|
|
448
|
+
LinearResidualBlock,
|
|
449
|
+
UNetLinear,
|
|
450
|
+
nn.Flatten,
|
|
451
|
+
nn.Linear,
|
|
452
|
+
nn.Conv1d,
|
|
453
|
+
nn.ReLU,
|
|
454
|
+
nn.BatchNorm1d,
|
|
455
|
+
nn.LayerNorm,
|
|
456
|
+
nn.ELU,
|
|
457
|
+
nn.PReLU,
|
|
458
|
+
nn.GLU,
|
|
459
|
+
nn.Unflatten,
|
|
460
|
+
]
|
|
461
|
+
)
|