DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,211 @@
1
+ from concurrent.futures import CancelledError, ProcessPoolExecutor
2
+ from concurrent.futures import ThreadPoolExecutor as _ThreadPoolExecutor
3
+ import multiprocessing
4
+ import os
5
+ from multiprocessing import get_context
6
+ import queue
7
+
8
+ from . import job_util
9
+
10
+ have_cloudpickle = False
11
+ cloudpickle = None
12
+ try:
13
+ import cloudpickle # type: ignore[reportMissingImports]
14
+
15
+ have_cloudpickle = True
16
+ except ImportError:
17
+ try:
18
+ from joblib.externals import cloudpickle
19
+
20
+ have_cloudpickle = True
21
+ except ImportError:
22
+ pass
23
+
24
+
25
+ class ThreadPoolExecutor(_ThreadPoolExecutor):
26
+ """shim for same api ([mp_]context args)"""
27
+
28
+ def __init__(
29
+ self,
30
+ max_workers=None,
31
+ mp_context=None,
32
+ initializer=None,
33
+ initargs=(),
34
+ context=None,
35
+ ):
36
+ super().__init__(
37
+ max_workers=max_workers, initializer=initializer, initargs=initargs
38
+ )
39
+
40
+
41
+ class MockFuture:
42
+ """A non-concurrent class for mocking the concurrent.futures API."""
43
+
44
+ def __init__(self, f, *args):
45
+ self.f = f
46
+ self.args = args
47
+
48
+ def result(self):
49
+ return self.f(*self.args)
50
+
51
+
52
+ class MockPoolExecutor:
53
+ """A non-concurrent class for mocking the concurrent.futures API."""
54
+
55
+ def __init__(
56
+ self,
57
+ max_workers=None,
58
+ mp_context=None,
59
+ initializer=None,
60
+ initargs=(),
61
+ context=None,
62
+ ):
63
+ self.initargs = initargs
64
+ self.initializer = initializer
65
+ self.initialized = False
66
+ self.cancelled = False
67
+
68
+ def _initialize(self):
69
+ if self.initialized:
70
+ return
71
+ if self.initializer is not None:
72
+ self.initializer(*self.initargs)
73
+ self.initialized = True
74
+
75
+ def map(self, function, *iterables, timeout=None, chunksize=1):
76
+ self._initialize()
77
+ for result in map(function, *iterables):
78
+ yield result
79
+ if self.cancelled:
80
+ raise CancelledError()
81
+
82
+ def __enter__(self):
83
+ return self
84
+
85
+ def __exit__(self, exc_type, exc_val, exc_tb):
86
+ return
87
+
88
+ def submit(self, f, *args):
89
+ self._initialize()
90
+ return MockFuture(f, *args)
91
+
92
+ def shutdown(self, wait=True, *, cancel_futures=False):
93
+ if cancel_futures:
94
+ self.cancelled = True
95
+
96
+
97
+ class MockQueue:
98
+ """Another helper class for turning off concurrency when debugging."""
99
+
100
+ def __init__(self):
101
+ self.q = []
102
+ self.put = self.q.append
103
+ self.get = lambda: self.q.pop(0)
104
+
105
+
106
+ def cloudpickle_run(fn, args):
107
+ assert cloudpickle is not None
108
+ fn = cloudpickle.loads(fn)
109
+ args, kwargs = cloudpickle.loads(args)
110
+ # return cloudpickle.dumps(fn(*args, **kwargs))
111
+ return fn(*args, **kwargs)
112
+
113
+
114
+ # def uncloudpickle(future):
115
+ # res = future.result()
116
+ # future._result = cloudpickle.loads(res)
117
+
118
+
119
+ class CloudpicklePoolExecutor(ProcessPoolExecutor):
120
+ def submit(self, fn, /, *args, **kwargs):
121
+ assert cloudpickle is not None
122
+ args = cloudpickle.dumps((args, kwargs))
123
+ future = super().submit(cloudpickle_run, cloudpickle.dumps(fn), args)
124
+ # future.add_done_callback(uncloudpickle_callback)
125
+ return future
126
+
127
+
128
+ def rank_init(queue):
129
+ rank_init.rank = queue.get() # type: ignore
130
+
131
+
132
+ def pool_from_cfg(computation_config=None, with_rank_queue=False, check_local=False):
133
+ if computation_config is None:
134
+ computation_config = job_util.get_global_computation_config()
135
+
136
+ return get_pool(
137
+ computation_config.actual_n_jobs(),
138
+ cls=computation_config.executor,
139
+ with_rank_queue=with_rank_queue,
140
+ check_local=check_local,
141
+ multi_gpu=computation_config.is_multi_gpu(),
142
+ )
143
+
144
+
145
+ def get_pool(
146
+ n_jobs,
147
+ context="spawn",
148
+ cls="threading_unless_multigpu",
149
+ with_rank_queue=False,
150
+ rank_queue_empty=False,
151
+ n_tasks=None,
152
+ max_tasks_per_child=None,
153
+ check_local=False,
154
+ multi_gpu=False,
155
+ ):
156
+ if n_jobs == -1:
157
+ try:
158
+ n_jobs = os.process_cpu_count() # type: ignore
159
+ except AttributeError:
160
+ n_jobs = multiprocessing.cpu_count()
161
+ do_parallel = n_jobs >= 1
162
+ n_jobs = max(1, n_jobs)
163
+
164
+ if isinstance(cls, str):
165
+ if cls == "threading_unless_multigpu":
166
+ if (n_jobs > 1 and multi_gpu) or os.name == "nt":
167
+ cls = "ProcessPoolExecutor"
168
+ else:
169
+ cls = "ThreadPoolExecutor"
170
+ if cls == "CloudpicklePoolExecutor":
171
+ cls = CloudpicklePoolExecutor
172
+ elif cls == "ThreadPoolExecutor":
173
+ cls = ThreadPoolExecutor
174
+ elif cls == "ProcessPoolExecutor":
175
+ cls = ProcessPoolExecutor
176
+ elif cls == "MockPoolExecutor":
177
+ cls = MockPoolExecutor
178
+ else:
179
+ assert False
180
+
181
+ if cls == CloudpicklePoolExecutor and not have_cloudpickle:
182
+ cls = ProcessPoolExecutor
183
+
184
+ Executor = cls if do_parallel else MockPoolExecutor
185
+ is_local = cls in (MockPoolExecutor, ThreadPoolExecutor)
186
+ context = get_context(context)
187
+
188
+ if with_rank_queue:
189
+ if do_parallel and is_local:
190
+ rank_queue = queue.SimpleQueue()
191
+ elif do_parallel:
192
+ manager = context.Manager()
193
+ rank_queue = manager.Queue()
194
+ else:
195
+ rank_queue = MockQueue()
196
+
197
+ if not rank_queue_empty:
198
+ n_repeats = 1
199
+ if max_tasks_per_child is not None:
200
+ n_repeats = n_tasks // max_tasks_per_child + 1
201
+ for _ in range(n_repeats):
202
+ for rank in range(n_jobs):
203
+ rank_queue.put(rank)
204
+
205
+ if check_local:
206
+ return n_jobs, Executor, context, rank_queue, is_local
207
+ return n_jobs, Executor, context, rank_queue
208
+
209
+ if check_local:
210
+ return n_jobs, Executor, context, is_local
211
+ return n_jobs, Executor, context
@@ -0,0 +1,461 @@
1
+ import torch
2
+ import torch.nn.functional as F
3
+ from torch import nn
4
+
5
+
6
+ class AttentionBlock(nn.Module):
7
+ def __init__(self, embed_dim, num_heads=1, dropout=0.0):
8
+ super().__init__()
9
+ self.embed_dim = embed_dim
10
+ self.num_heads = num_heads
11
+ self.scaling = embed_dim**-0.5
12
+
13
+ self.q_proj = nn.Linear(embed_dim, embed_dim)
14
+ self.k_proj = nn.Linear(embed_dim, embed_dim)
15
+ self.v_proj = nn.Linear(embed_dim, embed_dim)
16
+ self.out_proj = nn.Linear(embed_dim, embed_dim)
17
+ self.dropout = nn.Dropout(dropout)
18
+
19
+ def forward(self, x):
20
+ # x: [batch_size, seq_length, embed_dim]
21
+ q = self.q_proj(x) # [batch_size, seq_length, embed_dim]
22
+ k = self.k_proj(x) # [batch_size, seq_length, embed_dim]
23
+ v = self.v_proj(x) # [batch_size, seq_length, embed_dim]
24
+
25
+ # Compute attention scores
26
+ attn_scores = torch.matmul(q, k.transpose(-2, -1)) * self.scaling
27
+ # [batch_size, seq_length, seq_length]
28
+ attn_weights = torch.softmax(attn_scores, dim=-1)
29
+ attn_weights = self.dropout(attn_weights)
30
+
31
+ # Apply attention weights
32
+ # [batch_size, seq_length, embed_dim]
33
+ attn_output = torch.matmul(attn_weights, v)
34
+ return self.out_proj(attn_output)
35
+
36
+
37
+ class ResidualBlock(nn.Module):
38
+ def __init__(self, input_dim, output_dim, norm_kind="batchnorm"):
39
+ super().__init__()
40
+ self.input_dim = input_dim
41
+ self.output_dim = output_dim
42
+ self.norm_kind = norm_kind
43
+
44
+ self.linear = nn.Linear(input_dim, output_dim)
45
+ self.norm = get_norm(output_dim, norm_kind)
46
+ self.relu = nn.ReLU()
47
+ self.projection = None
48
+ if input_dim != output_dim:
49
+ self.projection = nn.Linear(input_dim, output_dim)
50
+
51
+ def forward(self, x):
52
+ residual = x
53
+ out = self.linear(x)
54
+ if self.norm is not None:
55
+ out = self.norm(out)
56
+ out = self.relu(out)
57
+ if self.projection is not None:
58
+ residual = self.projection(residual)
59
+ return out + residual
60
+
61
+
62
+ class ConcatResidualBlock(nn.Module):
63
+ def __init__(self, input_dim, output_dim, norm_kind="batchnorm"):
64
+ super().__init__()
65
+ self.input_dim = input_dim
66
+ self.output_dim = output_dim
67
+ self.norm_kind = norm_kind
68
+
69
+ self.linear = nn.Linear(input_dim, output_dim)
70
+ self.project_back = nn.Linear(input_dim + output_dim, output_dim)
71
+ self.norm = get_norm(output_dim, norm_kind)
72
+
73
+ self.relu = nn.ReLU()
74
+
75
+ def forward(self, x):
76
+ out = self.linear(x)
77
+ if self.norm is not None:
78
+ out = self.norm(out)
79
+ out = self.relu(out)
80
+
81
+ concatenated = torch.cat([x, out], dim=-1)
82
+ projected = self.project_back(concatenated)
83
+ return projected
84
+
85
+
86
+ class LinearResidualBlock(nn.Module):
87
+ def __init__(self, input_dim, hidden_dim, norm_kind=None):
88
+ super().__init__()
89
+ self.input_dim = input_dim
90
+ self.hidden_dim = hidden_dim
91
+ self.norm_kind = norm_kind
92
+
93
+ self.linear1 = nn.Linear(input_dim, hidden_dim)
94
+ self.linear2 = nn.Linear(hidden_dim, input_dim)
95
+ self.norm = get_norm(hidden_dim, norm_kind)
96
+
97
+ self.relu = nn.ReLU()
98
+
99
+ def forward(self, x):
100
+ out = self.linear1(x)
101
+ if self.norm is not None:
102
+ out = self.norm(out)
103
+ out = self.relu(out)
104
+ out = self.linear2(out)
105
+ return out + x
106
+
107
+
108
+ class UNetLinear(nn.Module):
109
+ def __init__(
110
+ self,
111
+ input_dim,
112
+ half_hidden_dims,
113
+ nonlinearity,
114
+ norm_kind=None,
115
+ ):
116
+ self.blocks_in = nn.ModuleList()
117
+ self.blocks_out = nn.ModuleList()
118
+
119
+ inner_gated = nonlinearity = torch.nn.GLU
120
+
121
+ *in_dims, middle_dim = half_hidden_dims
122
+ out_dims = in_dims[::-1]
123
+
124
+ current_dim = input_dim
125
+ out_dims_in = []
126
+ for out_dim in half_hiddden_dims:
127
+ block_layers = []
128
+ block_layers.append(nn.Linear(current_dim, (1 + inner_gated) * out_dim))
129
+ norm = get_norm((1 + inner_gated) * out_dim, norm_kind)
130
+ if norm is not None:
131
+ block_layers.append(norm)
132
+ block_layers.append(nonlinearity())
133
+ self.blocks_in.append(nn.Sequential(*block_layers))
134
+ current_dim = out_dim
135
+ out_dims_in.append(out_dim)
136
+
137
+ block_layers = []
138
+ block_layers.append(nn.Linear(current_dim, (1 + inner_gated) * middle_dim))
139
+ norm = get_norm((1 + inner_gated) * middle_dim, norm_kind)
140
+ if norm is not None:
141
+ block_layers.append(norm)
142
+ block_layers.append(nonlinearity())
143
+ self.middle_block = nn.Sequential(*block_layers)
144
+ current_dim = middle_dim
145
+
146
+ for out_dim in out_dims:
147
+ cat_dim = out_dims_in.pop()
148
+ current_dim = current_dim + cat_dim
149
+
150
+ block_layers = [Cat()]
151
+ block_layers.append(nn.Linear(current_dim, (1 + inner_gated) * out_dim))
152
+ norm = get_norm((1 + inner_gated) * out_dim, norm_kind)
153
+ if norm is not None:
154
+ block_layers.append(norm)
155
+ block_layers.append(nonlinearity())
156
+ self.blocks_out.append(nn.Sequential(*block_layers))
157
+ current_dim = out_dim
158
+ self.final_dim = current_dim
159
+ assert not out_dims_in
160
+
161
+ def forward(self, x):
162
+ stack = []
163
+ for b in self.blocks_in:
164
+ x = b(x)
165
+ stack.append(x)
166
+ x = self.middle_block(x)
167
+ for b in self.blocks_out:
168
+ x = b((x, stack.pop()))
169
+ return x
170
+
171
+
172
+ def get_mlp(
173
+ input_dim,
174
+ hidden_dims,
175
+ output_dim,
176
+ norm_kind="batchnorm",
177
+ res_type="none",
178
+ output_layer="linear",
179
+ nonlinearity="ReLU",
180
+ attention_layer=False,
181
+ num_heads=4,
182
+ ):
183
+ layers = []
184
+ if isinstance(nonlinearity, str):
185
+ nonlinearity = getattr(torch.nn, nonlinearity)
186
+ gated = nonlinearity == torch.nn.GLU
187
+
188
+ if res_type == "blocks_concat":
189
+ current_dim = input_dim
190
+ for out_dim in hidden_dims:
191
+ layers.append(ConcatResidualBlock(current_dim, out_dim, norm_kind))
192
+ current_dim = out_dim
193
+ if attention_layer:
194
+ layers.append(AttentionBlock(current_dim, num_heads=num_heads))
195
+ final_dim = current_dim
196
+
197
+ elif res_type == "blocks_add":
198
+ input_dims = [input_dim, *hidden_dims[:-1]]
199
+ for ind, outd in zip(input_dims, hidden_dims):
200
+ layers.append(ResidualBlock(ind, outd, norm_kind))
201
+ if attention_layer:
202
+ layers.append(AttentionBlock(outd, num_heads=num_heads))
203
+ final_dim = hidden_dims[-1] if hidden_dims else input_dim
204
+
205
+ elif res_type == "blocks_linear":
206
+ assert len(hidden_dims) > 1
207
+ compression_dim = hidden_dims[0]
208
+ layers.append(nn.Linear(input_dim, compression_dim))
209
+ for hd in hidden_dims[1:]:
210
+ layers.append(LinearResidualBlock(compression_dim, hd, norm_kind=norm_kind))
211
+ final_dim = compression_dim
212
+
213
+ elif res_type == "unet_linear":
214
+ layers.append(UNetLinear(input_dim, hidden_dims, out_dim, nonlinearity))
215
+ final_dim = layers[-1].final_dim
216
+
217
+ elif res_type in ("none", "outer"):
218
+ # res_type == "none" is handled in get_waveform_mlp
219
+ # res_type in get_mlp is the *inner* residual type in each block
220
+ current_dim = input_dim
221
+ for out_dim in hidden_dims:
222
+ layers.append(nn.Linear(current_dim, (1 + gated) * out_dim))
223
+ norm = get_norm((1 + gated) * out_dim, norm_kind)
224
+ if norm is not None:
225
+ layers.append(norm)
226
+ layers.append(nonlinearity())
227
+ current_dim = out_dim
228
+ if attention_layer:
229
+ layers.append(AttentionBlock(current_dim, num_heads=num_heads))
230
+ final_dim = current_dim
231
+
232
+ else:
233
+ raise ValueError(f"Unsupported res_type: {res_type}")
234
+
235
+ if output_layer == "linear":
236
+ layers.append(nn.Linear(final_dim, output_dim))
237
+ elif output_layer == "gated_linear":
238
+ layers.append(nn.Linear(final_dim, 2 * output_dim))
239
+ layers.append(nn.GLU())
240
+ else:
241
+ assert False
242
+
243
+ return nn.Sequential(*layers)
244
+
245
+
246
+ def get_waveform_mlp(
247
+ spike_length_samples,
248
+ n_input_channels,
249
+ hidden_dims,
250
+ output_dim,
251
+ input_includes_mask=True,
252
+ norm_kind="batchnorm",
253
+ scaling=None,
254
+ log_transform=False,
255
+ channelwise_dropout_p=0.0,
256
+ separated_mask_input=False,
257
+ initial_conv_fullheight=False,
258
+ final_conv_fullheight=False,
259
+ return_initial_shape=False,
260
+ res_type="none",
261
+ output_layer="linear",
262
+ nonlinearity="ReLU",
263
+ attention_layer=False,
264
+ num_heads=4,
265
+ ):
266
+ input_dim = n_input_channels * (spike_length_samples + input_includes_mask)
267
+ layers = []
268
+ if log_transform:
269
+ layers.append(WaveformOnly(LogTransform()))
270
+
271
+ if initial_conv_fullheight:
272
+ # what Conv1d considers channels is actually time (conv1d is ncl).
273
+ # so this is matmul over time, and kernel size is 1 to be separate over chans
274
+ conv = nn.Conv1d(spike_length_samples, spike_length_samples, kernel_size=1)
275
+ layers.append(WaveformOnly(conv))
276
+ norm = get_norm(n_input_channels, norm_kind)
277
+ if norm is not None:
278
+ layers.append(
279
+ WaveformOnly(nn.Sequential(Permute(0, 2, 1), norm, Permute(0, 2, 1)))
280
+ )
281
+ layers.append(WaveformOnly(nn.ReLU()))
282
+
283
+ if separated_mask_input:
284
+ layers.append(Cat(dim=1))
285
+
286
+ if channelwise_dropout_p:
287
+ layers.append(ChannelwiseDropout(channelwise_dropout_p))
288
+
289
+ layers.append(nn.Flatten())
290
+ mlp = get_mlp(
291
+ input_dim,
292
+ hidden_dims,
293
+ output_dim,
294
+ norm_kind=norm_kind,
295
+ res_type=res_type,
296
+ nonlinearity=nonlinearity,
297
+ attention_layer=attention_layer,
298
+ output_layer=output_layer,
299
+ num_heads=num_heads,
300
+ )
301
+ layers.append(mlp)
302
+
303
+ if return_initial_shape:
304
+ layers.append(nn.Unflatten(-1, (spike_length_samples, n_input_channels)))
305
+ if final_conv_fullheight:
306
+ norm = get_norm(n_input_channels, norm_kind)
307
+ if norm is not None:
308
+ layers.append(nn.Sequential(Permute(0, 2, 1), norm, Permute(0, 2, 1)))
309
+ layers.append(nn.ReLU())
310
+ conv = nn.Conv1d(spike_length_samples, spike_length_samples, kernel_size=1)
311
+ layers.append(conv)
312
+
313
+ net = nn.Sequential(*layers)
314
+
315
+ if scaling not in (None, "none"):
316
+ net = Rescaling(net, scaling, waveform_only=True)
317
+
318
+ if res_type == "outer":
319
+ net = WaveformOnlyResidualForm(net)
320
+
321
+ if log_transform:
322
+ layers.append(WaveformOnly(ExpTransform()))
323
+
324
+ return net
325
+
326
+
327
+ def get_norm(n_features, norm_kind=None):
328
+ if norm_kind == "batchnorm":
329
+ return nn.BatchNorm1d(n_features)
330
+ if norm_kind == "layernorm":
331
+ return nn.LayerNorm(n_features)
332
+ assert norm_kind in ("none", None)
333
+ return None
334
+
335
+
336
+ class ResidualForm(nn.Module):
337
+
338
+ def __init__(self, module):
339
+ super().__init__()
340
+ self.module = module
341
+
342
+ def forward(self, input):
343
+ output = self.module(input)
344
+ return input + output
345
+
346
+
347
+ class Rescaling(nn.Module):
348
+ def __init__(self, module, scale_kind="std", waveform_only=False):
349
+ super().__init__()
350
+ self.scale_kind = scale_kind
351
+ self.module = module
352
+ self.waveform_only = waveform_only
353
+
354
+ def forward(self, input):
355
+ if self.waveform_only:
356
+ input, masks = input
357
+ dim = tuple(range(1, input.ndim))
358
+ if self.scale_kind == "max":
359
+ scales = input.abs().amax(dim=dim, keepdim=True)
360
+ elif self.scale_kind == "std":
361
+ scales = input.std(dim=dim, keepdim=True)
362
+ else:
363
+ assert False
364
+ if self.waveform_only:
365
+ output = self.module((input / scales, masks))
366
+ else:
367
+ output = self.module(input / scales)
368
+ return output * scales
369
+
370
+
371
+ class LogTransform(nn.Module):
372
+ def forward(self, input):
373
+ sgn = torch.sign(input)
374
+ log = torch.log1p(input.abs())
375
+ return log * sgn
376
+
377
+
378
+ class ExpTransform(nn.Module):
379
+ def forward(self, input):
380
+ sgn = torch.sign(input)
381
+ exp = torch.expm1(input.abs())
382
+ return exp * sgn
383
+
384
+
385
+ class WaveformOnlyResidualForm(nn.Module):
386
+ def __init__(self, module):
387
+ super().__init__()
388
+ self.module = module
389
+
390
+ def forward(self, inputs):
391
+ waveforms, masks = inputs
392
+ output = self.module(inputs)
393
+ return waveforms + output
394
+
395
+
396
+ class ChannelwiseDropout(nn.Module):
397
+ def __init__(self, p):
398
+ super().__init__()
399
+ self.p = p
400
+
401
+ def forward(self, waveforms):
402
+ res = F.dropout1d(waveforms.permute(0, 2, 1), p=self.p, training=self.training)
403
+ res = res.permute(0, 2, 1)
404
+ return res
405
+
406
+
407
+ class Cat(nn.Module):
408
+ def __init__(self, dim=-1):
409
+ super().__init__()
410
+ self.dim = dim
411
+
412
+ def forward(self, inputs):
413
+ return torch.cat(inputs, dim=self.dim)
414
+
415
+
416
+ class Permute(nn.Module):
417
+ def __init__(self, *dims):
418
+ super().__init__()
419
+ self.dims = dims
420
+
421
+ def forward(self, inputs):
422
+ return inputs.permute(*self.dims)
423
+
424
+
425
+ class WaveformOnly(nn.Module):
426
+ def __init__(self, module):
427
+ super().__init__()
428
+ self.module = module
429
+
430
+ def forward(self, inputs):
431
+ waveforms, masks = inputs
432
+ return self.module(waveforms), masks
433
+
434
+
435
+ # is this what they want us to do??
436
+ if hasattr(torch.serialization, "add_safe_globals"):
437
+ torch.serialization.add_safe_globals(
438
+ [
439
+ ResidualForm,
440
+ WaveformOnlyResidualForm,
441
+ ChannelwiseDropout,
442
+ Cat,
443
+ Permute,
444
+ WaveformOnly,
445
+ Rescaling,
446
+ LogTransform,
447
+ ExpTransform,
448
+ LinearResidualBlock,
449
+ UNetLinear,
450
+ nn.Flatten,
451
+ nn.Linear,
452
+ nn.Conv1d,
453
+ nn.ReLU,
454
+ nn.BatchNorm1d,
455
+ nn.LayerNorm,
456
+ nn.ELU,
457
+ nn.PReLU,
458
+ nn.GLU,
459
+ nn.Unflatten,
460
+ ]
461
+ )