rapidtide 3.0.11__py3-none-any.whl → 3.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rapidtide/Colortables.py +492 -27
- rapidtide/OrthoImageItem.py +1049 -46
- rapidtide/RapidtideDataset.py +1533 -86
- rapidtide/_version.py +3 -3
- rapidtide/calccoherence.py +196 -29
- rapidtide/calcnullsimfunc.py +188 -40
- rapidtide/calcsimfunc.py +242 -42
- rapidtide/correlate.py +1203 -383
- rapidtide/data/examples/src/testLD +56 -0
- rapidtide/data/examples/src/testalign +1 -1
- rapidtide/data/examples/src/testdelayvar +0 -1
- rapidtide/data/examples/src/testfmri +53 -3
- rapidtide/data/examples/src/testglmfilt +5 -5
- rapidtide/data/examples/src/testhappy +29 -7
- rapidtide/data/examples/src/testppgproc +17 -0
- rapidtide/data/examples/src/testrolloff +11 -0
- rapidtide/data/models/model_cnn_pytorch/best_model.pth +0 -0
- rapidtide/data/models/model_cnn_pytorch/loss.png +0 -0
- rapidtide/data/models/model_cnn_pytorch/loss.txt +1 -0
- rapidtide/data/models/model_cnn_pytorch/model.pth +0 -0
- rapidtide/data/models/model_cnn_pytorch/model_meta.json +68 -0
- rapidtide/decorators.py +91 -0
- rapidtide/dlfilter.py +2226 -110
- rapidtide/dlfiltertorch.py +4842 -0
- rapidtide/externaltools.py +327 -12
- rapidtide/fMRIData_class.py +79 -40
- rapidtide/filter.py +1899 -810
- rapidtide/fit.py +2011 -581
- rapidtide/genericmultiproc.py +93 -18
- rapidtide/happy_supportfuncs.py +2047 -172
- rapidtide/helper_classes.py +584 -43
- rapidtide/io.py +2370 -372
- rapidtide/linfitfiltpass.py +346 -99
- rapidtide/makelaggedtcs.py +210 -24
- rapidtide/maskutil.py +448 -62
- rapidtide/miscmath.py +827 -121
- rapidtide/multiproc.py +210 -22
- rapidtide/patchmatch.py +242 -42
- rapidtide/peakeval.py +31 -31
- rapidtide/ppgproc.py +2203 -0
- rapidtide/qualitycheck.py +352 -39
- rapidtide/refinedelay.py +431 -57
- rapidtide/refineregressor.py +494 -189
- rapidtide/resample.py +671 -185
- rapidtide/scripts/applyppgproc.py +28 -0
- rapidtide/scripts/showxcorr_legacy.py +7 -7
- rapidtide/scripts/stupidramtricks.py +15 -17
- rapidtide/simFuncClasses.py +1052 -77
- rapidtide/simfuncfit.py +269 -69
- rapidtide/stats.py +540 -238
- rapidtide/tests/happycomp +9 -0
- rapidtide/tests/test_cleanregressor.py +1 -2
- rapidtide/tests/test_dlfiltertorch.py +627 -0
- rapidtide/tests/test_findmaxlag.py +24 -8
- rapidtide/tests/test_fullrunhappy_v1.py +0 -2
- rapidtide/tests/test_fullrunhappy_v2.py +0 -2
- rapidtide/tests/test_fullrunhappy_v3.py +11 -4
- rapidtide/tests/test_fullrunhappy_v4.py +10 -2
- rapidtide/tests/test_fullrunrapidtide_v7.py +1 -1
- rapidtide/tests/test_getparsers.py +11 -3
- rapidtide/tests/test_refinedelay.py +0 -1
- rapidtide/tests/test_simroundtrip.py +16 -8
- rapidtide/tests/test_stcorrelate.py +3 -1
- rapidtide/tests/utils.py +9 -8
- rapidtide/tidepoolTemplate.py +142 -38
- rapidtide/tidepoolTemplate_alt.py +165 -44
- rapidtide/tidepoolTemplate_big.py +189 -52
- rapidtide/util.py +1217 -118
- rapidtide/voxelData.py +684 -37
- rapidtide/wiener.py +136 -23
- rapidtide/wiener2.py +113 -7
- rapidtide/workflows/adjustoffset.py +105 -3
- rapidtide/workflows/aligntcs.py +85 -2
- rapidtide/workflows/applydlfilter.py +87 -10
- rapidtide/workflows/applyppgproc.py +540 -0
- rapidtide/workflows/atlasaverage.py +210 -47
- rapidtide/workflows/atlastool.py +100 -3
- rapidtide/workflows/calcSimFuncMap.py +288 -69
- rapidtide/workflows/calctexticc.py +201 -9
- rapidtide/workflows/ccorrica.py +101 -6
- rapidtide/workflows/cleanregressor.py +165 -31
- rapidtide/workflows/delayvar.py +171 -23
- rapidtide/workflows/diffrois.py +81 -3
- rapidtide/workflows/endtidalproc.py +144 -4
- rapidtide/workflows/fdica.py +195 -15
- rapidtide/workflows/filtnifti.py +70 -3
- rapidtide/workflows/filttc.py +74 -3
- rapidtide/workflows/fitSimFuncMap.py +202 -51
- rapidtide/workflows/fixtr.py +73 -3
- rapidtide/workflows/gmscalc.py +113 -3
- rapidtide/workflows/happy.py +801 -199
- rapidtide/workflows/happy2std.py +144 -12
- rapidtide/workflows/happy_parser.py +163 -23
- rapidtide/workflows/histnifti.py +118 -2
- rapidtide/workflows/histtc.py +84 -3
- rapidtide/workflows/linfitfilt.py +117 -4
- rapidtide/workflows/localflow.py +328 -28
- rapidtide/workflows/mergequality.py +79 -3
- rapidtide/workflows/niftidecomp.py +322 -18
- rapidtide/workflows/niftistats.py +174 -4
- rapidtide/workflows/pairproc.py +98 -4
- rapidtide/workflows/pairwisemergenifti.py +85 -2
- rapidtide/workflows/parser_funcs.py +1421 -40
- rapidtide/workflows/physiofreq.py +137 -11
- rapidtide/workflows/pixelcomp.py +207 -5
- rapidtide/workflows/plethquality.py +103 -21
- rapidtide/workflows/polyfitim.py +151 -11
- rapidtide/workflows/proj2flow.py +75 -2
- rapidtide/workflows/rankimage.py +111 -4
- rapidtide/workflows/rapidtide.py +368 -76
- rapidtide/workflows/rapidtide2std.py +98 -2
- rapidtide/workflows/rapidtide_parser.py +109 -9
- rapidtide/workflows/refineDelayMap.py +144 -33
- rapidtide/workflows/refineRegressor.py +675 -96
- rapidtide/workflows/regressfrommaps.py +161 -37
- rapidtide/workflows/resamplenifti.py +85 -3
- rapidtide/workflows/resampletc.py +91 -3
- rapidtide/workflows/retrolagtcs.py +99 -9
- rapidtide/workflows/retroregress.py +176 -26
- rapidtide/workflows/roisummarize.py +174 -5
- rapidtide/workflows/runqualitycheck.py +71 -3
- rapidtide/workflows/showarbcorr.py +149 -6
- rapidtide/workflows/showhist.py +86 -2
- rapidtide/workflows/showstxcorr.py +160 -3
- rapidtide/workflows/showtc.py +159 -3
- rapidtide/workflows/showxcorrx.py +190 -10
- rapidtide/workflows/showxy.py +185 -15
- rapidtide/workflows/simdata.py +264 -38
- rapidtide/workflows/spatialfit.py +77 -2
- rapidtide/workflows/spatialmi.py +250 -27
- rapidtide/workflows/spectrogram.py +305 -32
- rapidtide/workflows/synthASL.py +154 -3
- rapidtide/workflows/tcfrom2col.py +76 -2
- rapidtide/workflows/tcfrom3col.py +74 -2
- rapidtide/workflows/tidepool.py +2971 -130
- rapidtide/workflows/utils.py +19 -14
- rapidtide/workflows/utils_doc.py +293 -0
- rapidtide/workflows/variabilityizer.py +116 -3
- {rapidtide-3.0.11.dist-info → rapidtide-3.1.1.dist-info}/METADATA +10 -8
- {rapidtide-3.0.11.dist-info → rapidtide-3.1.1.dist-info}/RECORD +144 -128
- {rapidtide-3.0.11.dist-info → rapidtide-3.1.1.dist-info}/entry_points.txt +1 -0
- {rapidtide-3.0.11.dist-info → rapidtide-3.1.1.dist-info}/WHEEL +0 -0
- {rapidtide-3.0.11.dist-info → rapidtide-3.1.1.dist-info}/licenses/LICENSE +0 -0
- {rapidtide-3.0.11.dist-info → rapidtide-3.1.1.dist-info}/top_level.txt +0 -0
rapidtide/dlfilter.py
CHANGED
|
@@ -26,6 +26,7 @@ import warnings
|
|
|
26
26
|
import matplotlib as mpl
|
|
27
27
|
import matplotlib.pyplot as plt
|
|
28
28
|
import numpy as np
|
|
29
|
+
from numpy.typing import NDArray
|
|
29
30
|
|
|
30
31
|
with warnings.catch_warnings():
|
|
31
32
|
warnings.simplefilter("ignore")
|
|
@@ -120,32 +121,186 @@ class DeepLearningFilter:
|
|
|
120
121
|
|
|
121
122
|
def __init__(
|
|
122
123
|
self,
|
|
123
|
-
window_size=128,
|
|
124
|
-
num_layers=5,
|
|
125
|
-
dropout_rate=0.3,
|
|
126
|
-
num_pretrain_epochs=0,
|
|
127
|
-
num_epochs=1,
|
|
128
|
-
activation="relu",
|
|
129
|
-
modelroot=".",
|
|
130
|
-
dofft=False,
|
|
131
|
-
excludethresh=4.0,
|
|
132
|
-
usebadpts=False,
|
|
133
|
-
thesuffix="25.0Hz",
|
|
134
|
-
modelpath=".",
|
|
135
|
-
thedatadir="/Users/frederic/Documents/MR_data/physioconn/timecourses",
|
|
136
|
-
inputfrag="abc",
|
|
137
|
-
targetfrag="xyz",
|
|
138
|
-
corrthresh=0.5,
|
|
139
|
-
excludebysubject=True,
|
|
140
|
-
startskip=200,
|
|
141
|
-
endskip=200,
|
|
142
|
-
step=1,
|
|
143
|
-
namesuffix=None,
|
|
144
|
-
readlim=None,
|
|
145
|
-
readskip=None,
|
|
146
|
-
countlim=None,
|
|
124
|
+
window_size: int = 128,
|
|
125
|
+
num_layers: int = 5,
|
|
126
|
+
dropout_rate: float = 0.3,
|
|
127
|
+
num_pretrain_epochs: int = 0,
|
|
128
|
+
num_epochs: int = 1,
|
|
129
|
+
activation: str = "relu",
|
|
130
|
+
modelroot: str = ".",
|
|
131
|
+
dofft: bool = False,
|
|
132
|
+
excludethresh: float = 4.0,
|
|
133
|
+
usebadpts: bool = False,
|
|
134
|
+
thesuffix: str = "25.0Hz",
|
|
135
|
+
modelpath: str = ".",
|
|
136
|
+
thedatadir: str = "/Users/frederic/Documents/MR_data/physioconn/timecourses",
|
|
137
|
+
inputfrag: str = "abc",
|
|
138
|
+
targetfrag: str = "xyz",
|
|
139
|
+
corrthresh: float = 0.5,
|
|
140
|
+
excludebysubject: bool = True,
|
|
141
|
+
startskip: int = 200,
|
|
142
|
+
endskip: int = 200,
|
|
143
|
+
step: int = 1,
|
|
144
|
+
namesuffix: str | None = None,
|
|
145
|
+
readlim: int | None = None,
|
|
146
|
+
readskip: int | None = None,
|
|
147
|
+
countlim: int | None = None,
|
|
147
148
|
**kwargs,
|
|
148
|
-
):
|
|
149
|
+
) -> None:
|
|
150
|
+
"""
|
|
151
|
+
Initialize the DeepLearningFilter with specified parameters.
|
|
152
|
+
|
|
153
|
+
This constructor sets up the configuration for a deep learning model used
|
|
154
|
+
for filtering physiological timecourses. It initializes various hyperparameters,
|
|
155
|
+
paths, and flags that control the behavior of the model and data processing.
|
|
156
|
+
|
|
157
|
+
Parameters
|
|
158
|
+
----------
|
|
159
|
+
window_size : int, optional
|
|
160
|
+
Size of the sliding window used for processing time series data. Default is 128.
|
|
161
|
+
num_layers : int, optional
|
|
162
|
+
Number of layers in the neural network model. Default is 5.
|
|
163
|
+
dropout_rate : float, optional
|
|
164
|
+
Dropout rate for regularization during training. Default is 0.3.
|
|
165
|
+
num_pretrain_epochs : int, optional
|
|
166
|
+
Number of pre-training epochs. Default is 0.
|
|
167
|
+
num_epochs : int, optional
|
|
168
|
+
Number of training epochs. Default is 1.
|
|
169
|
+
activation : str, optional
|
|
170
|
+
Activation function to use in the model. Default is "relu".
|
|
171
|
+
modelroot : str, optional
|
|
172
|
+
Root directory for model storage. Default is ".".
|
|
173
|
+
dofft : bool, optional
|
|
174
|
+
Whether to apply FFT transformation to input data. Default is False.
|
|
175
|
+
excludethresh : float, optional
|
|
176
|
+
Threshold for excluding data points based on correlation. Default is 4.0.
|
|
177
|
+
usebadpts : bool, optional
|
|
178
|
+
Whether to include bad points in the input. Default is False.
|
|
179
|
+
thesuffix : str, optional
|
|
180
|
+
Suffix to append to filenames. Default is "25.0Hz".
|
|
181
|
+
modelpath : str, optional
|
|
182
|
+
Path to save or load the model. Default is ".".
|
|
183
|
+
thedatadir : str, optional
|
|
184
|
+
Directory containing the physiological data files. Default is
|
|
185
|
+
"/Users/frederic/Documents/MR_data/physioconn/timecourses".
|
|
186
|
+
inputfrag : str, optional
|
|
187
|
+
Fragment identifier for input data. Default is "abc".
|
|
188
|
+
targetfrag : str, optional
|
|
189
|
+
Fragment identifier for target data. Default is "xyz".
|
|
190
|
+
corrthresh : float, optional
|
|
191
|
+
Correlation threshold for filtering. Default is 0.5.
|
|
192
|
+
excludebysubject : bool, optional
|
|
193
|
+
Whether to exclude data by subject. Default is True.
|
|
194
|
+
startskip : int, optional
|
|
195
|
+
Number of samples to skip at the beginning of each timecourse. Default is 200.
|
|
196
|
+
endskip : int, optional
|
|
197
|
+
Number of samples to skip at the end of each timecourse. Default is 200.
|
|
198
|
+
step : int, optional
|
|
199
|
+
Step size for sliding window. Default is 1.
|
|
200
|
+
namesuffix : str, optional
|
|
201
|
+
Suffix to append to model name. Default is None.
|
|
202
|
+
readlim : int, optional
|
|
203
|
+
Limit on number of samples to read. Default is None.
|
|
204
|
+
readskip : int, optional
|
|
205
|
+
Number of samples to skip when reading data. Default is None.
|
|
206
|
+
countlim : int, optional
|
|
207
|
+
Limit on number of timecourses to process. Default is None.
|
|
208
|
+
**kwargs
|
|
209
|
+
Additional keyword arguments passed to the parent class.
|
|
210
|
+
|
|
211
|
+
Notes
|
|
212
|
+
-----
|
|
213
|
+
The `inputsize` is dynamically set based on the `usebadpts` flag:
|
|
214
|
+
- If `usebadpts` is True, input size is 2.
|
|
215
|
+
- Otherwise, input size is 1.
|
|
216
|
+
|
|
217
|
+
Examples
|
|
218
|
+
--------
|
|
219
|
+
>>> filter = DeepLearningFilter(
|
|
220
|
+
... window_size=256,
|
|
221
|
+
... num_layers=6,
|
|
222
|
+
... dropout_rate=0.2,
|
|
223
|
+
... modelroot="/models",
|
|
224
|
+
... dofft=True
|
|
225
|
+
... )
|
|
226
|
+
"""
|
|
227
|
+
"""
|
|
228
|
+
Initialize the DeepLearningFilter with specified parameters.
|
|
229
|
+
|
|
230
|
+
This constructor sets up the configuration for a deep learning model used
|
|
231
|
+
for filtering physiological timecourses. It initializes various hyperparameters,
|
|
232
|
+
paths, and flags that control the behavior of the model and data processing.
|
|
233
|
+
|
|
234
|
+
Parameters
|
|
235
|
+
----------
|
|
236
|
+
window_size : int, optional
|
|
237
|
+
Size of the sliding window used for processing time series data. Default is 128.
|
|
238
|
+
num_layers : int, optional
|
|
239
|
+
Number of layers in the neural network model. Default is 5.
|
|
240
|
+
dropout_rate : float, optional
|
|
241
|
+
Dropout rate for regularization during training. Default is 0.3.
|
|
242
|
+
num_pretrain_epochs : int, optional
|
|
243
|
+
Number of pre-training epochs. Default is 0.
|
|
244
|
+
num_epochs : int, optional
|
|
245
|
+
Number of training epochs. Default is 1.
|
|
246
|
+
activation : str, optional
|
|
247
|
+
Activation function to use in the model. Default is "relu".
|
|
248
|
+
modelroot : str, optional
|
|
249
|
+
Root directory for model storage. Default is ".".
|
|
250
|
+
dofft : bool, optional
|
|
251
|
+
Whether to apply FFT transformation to input data. Default is False.
|
|
252
|
+
excludethresh : float, optional
|
|
253
|
+
Threshold for excluding data points based on correlation. Default is 4.0.
|
|
254
|
+
usebadpts : bool, optional
|
|
255
|
+
Whether to include bad points in the input. Default is False.
|
|
256
|
+
thesuffix : str, optional
|
|
257
|
+
Suffix to append to filenames. Default is "25.0Hz".
|
|
258
|
+
modelpath : str, optional
|
|
259
|
+
Path to save or load the model. Default is ".".
|
|
260
|
+
thedatadir : str, optional
|
|
261
|
+
Directory containing the physiological data files. Default is
|
|
262
|
+
"/Users/frederic/Documents/MR_data/physioconn/timecourses".
|
|
263
|
+
inputfrag : str, optional
|
|
264
|
+
Fragment identifier for input data. Default is "abc".
|
|
265
|
+
targetfrag : str, optional
|
|
266
|
+
Fragment identifier for target data. Default is "xyz".
|
|
267
|
+
corrthresh : float, optional
|
|
268
|
+
Correlation threshold for filtering. Default is 0.5.
|
|
269
|
+
excludebysubject : bool, optional
|
|
270
|
+
Whether to exclude data by subject. Default is True.
|
|
271
|
+
startskip : int, optional
|
|
272
|
+
Number of samples to skip at the beginning of each timecourse. Default is 200.
|
|
273
|
+
endskip : int, optional
|
|
274
|
+
Number of samples to skip at the end of each timecourse. Default is 200.
|
|
275
|
+
step : int, optional
|
|
276
|
+
Step size for sliding window. Default is 1.
|
|
277
|
+
namesuffix : str, optional
|
|
278
|
+
Suffix to append to model name. Default is None.
|
|
279
|
+
readlim : int, optional
|
|
280
|
+
Limit on number of samples to read. Default is None.
|
|
281
|
+
readskip : int, optional
|
|
282
|
+
Number of samples to skip when reading data. Default is None.
|
|
283
|
+
countlim : int, optional
|
|
284
|
+
Limit on number of timecourses to process. Default is None.
|
|
285
|
+
**kwargs
|
|
286
|
+
Additional keyword arguments passed to the parent class.
|
|
287
|
+
|
|
288
|
+
Notes
|
|
289
|
+
-----
|
|
290
|
+
The `inputsize` is dynamically set based on the `usebadpts` flag:
|
|
291
|
+
- If `usebadpts` is True, input size is 2.
|
|
292
|
+
- Otherwise, input size is 1.
|
|
293
|
+
|
|
294
|
+
Examples
|
|
295
|
+
--------
|
|
296
|
+
>>> filter = DeepLearningFilter(
|
|
297
|
+
... window_size=256,
|
|
298
|
+
... num_layers=6,
|
|
299
|
+
... dropout_rate=0.2,
|
|
300
|
+
... modelroot="/models",
|
|
301
|
+
... dofft=True
|
|
302
|
+
... )
|
|
303
|
+
"""
|
|
149
304
|
self.window_size = window_size
|
|
150
305
|
self.dropout_rate = dropout_rate
|
|
151
306
|
self.num_pretrain_epochs = num_pretrain_epochs
|
|
@@ -195,7 +350,91 @@ class DeepLearningFilter:
|
|
|
195
350
|
self.infodict["step"] = self.step
|
|
196
351
|
self.infodict["train_arch"] = sys.platform
|
|
197
352
|
|
|
198
|
-
def loaddata(self):
|
|
353
|
+
def loaddata(self) -> None:
|
|
354
|
+
"""
|
|
355
|
+
Load and preprocess data for training and validation.
|
|
356
|
+
|
|
357
|
+
This method initializes the data loading process by calling the `prep` function
|
|
358
|
+
with a set of parameters derived from the instance attributes. It handles both
|
|
359
|
+
FFT and non-FFT modes of data preprocessing. The loaded data is stored in
|
|
360
|
+
instance variables for use in subsequent training steps.
|
|
361
|
+
|
|
362
|
+
Parameters
|
|
363
|
+
----------
|
|
364
|
+
self : object
|
|
365
|
+
The instance of the class containing the following attributes:
|
|
366
|
+
- initialized : bool
|
|
367
|
+
Indicates whether the model has been initialized.
|
|
368
|
+
- dofft : bool
|
|
369
|
+
Whether to apply FFT transformation to the data.
|
|
370
|
+
- window_size : int
|
|
371
|
+
Size of the sliding window used for data segmentation.
|
|
372
|
+
- thesuffix : str
|
|
373
|
+
Suffix to append to filenames when reading data.
|
|
374
|
+
- thedatadir : str
|
|
375
|
+
Directory path where the data files are located.
|
|
376
|
+
- inputfrag : str
|
|
377
|
+
Fragment identifier for input data.
|
|
378
|
+
- targetfrag : str
|
|
379
|
+
Fragment identifier for target data.
|
|
380
|
+
- startskip : int
|
|
381
|
+
Number of samples to skip at the beginning of each file.
|
|
382
|
+
- endskip : int
|
|
383
|
+
Number of samples to skip at the end of each file.
|
|
384
|
+
- corrthresh : float
|
|
385
|
+
Correlation threshold for filtering data.
|
|
386
|
+
- step : int
|
|
387
|
+
Step size for sliding window.
|
|
388
|
+
- usebadpts : bool
|
|
389
|
+
Whether to include bad points in the data.
|
|
390
|
+
- excludethresh : float
|
|
391
|
+
Threshold for excluding data points.
|
|
392
|
+
- excludebysubject : bool
|
|
393
|
+
Whether to exclude data by subject.
|
|
394
|
+
- readlim : int
|
|
395
|
+
Limit on the number of samples to read.
|
|
396
|
+
- readskip : int
|
|
397
|
+
Number of samples to skip while reading.
|
|
398
|
+
- countlim : int
|
|
399
|
+
Limit on the number of data points to process.
|
|
400
|
+
|
|
401
|
+
Returns
|
|
402
|
+
-------
|
|
403
|
+
None
|
|
404
|
+
This method does not return any value. It modifies the instance attributes
|
|
405
|
+
in place.
|
|
406
|
+
|
|
407
|
+
Raises
|
|
408
|
+
------
|
|
409
|
+
Exception
|
|
410
|
+
If the model is not initialized prior to calling this method.
|
|
411
|
+
|
|
412
|
+
Notes
|
|
413
|
+
-----
|
|
414
|
+
The method assigns the following attributes to the instance after loading:
|
|
415
|
+
- train_x : array-like
|
|
416
|
+
Training input data.
|
|
417
|
+
- train_y : array-like
|
|
418
|
+
Training target data.
|
|
419
|
+
- val_x : array-like
|
|
420
|
+
Validation input data.
|
|
421
|
+
- val_y : array-like
|
|
422
|
+
Validation target data.
|
|
423
|
+
- Ns : int
|
|
424
|
+
Number of samples.
|
|
425
|
+
- tclen : int
|
|
426
|
+
Length of time series.
|
|
427
|
+
- thebatchsize : int
|
|
428
|
+
Batch size for training.
|
|
429
|
+
|
|
430
|
+
Examples
|
|
431
|
+
--------
|
|
432
|
+
>>> model = MyModel()
|
|
433
|
+
>>> model.initialized = True
|
|
434
|
+
>>> model.loaddata()
|
|
435
|
+
>>> print(model.train_x.shape)
|
|
436
|
+
(1000, 10)
|
|
437
|
+
"""
|
|
199
438
|
if not self.initialized:
|
|
200
439
|
raise Exception("model must be initialized prior to loading data")
|
|
201
440
|
|
|
@@ -257,10 +496,87 @@ class DeepLearningFilter:
|
|
|
257
496
|
)
|
|
258
497
|
|
|
259
498
|
@tf.function
|
|
260
|
-
def predict_model(self, X):
|
|
499
|
+
def predict_model(self, X: NDArray) -> NDArray:
|
|
500
|
+
"""
|
|
501
|
+
Make predictions using the trained model.
|
|
502
|
+
|
|
503
|
+
Parameters
|
|
504
|
+
----------
|
|
505
|
+
X : NDArray
|
|
506
|
+
Input features for prediction. Shape should be (n_samples, n_features)
|
|
507
|
+
where n_samples is the number of samples and n_features is the number
|
|
508
|
+
of features expected by the model.
|
|
509
|
+
|
|
510
|
+
Returns
|
|
511
|
+
-------
|
|
512
|
+
NDArray
|
|
513
|
+
Model predictions. Shape will depend on the specific model type but
|
|
514
|
+
typically follows (n_samples,) for regression or (n_samples, n_classes)
|
|
515
|
+
for classification.
|
|
516
|
+
|
|
517
|
+
Notes
|
|
518
|
+
-----
|
|
519
|
+
This method sets the model to inference mode by calling with training=False.
|
|
520
|
+
The predictions are made without computing gradients, making it efficient
|
|
521
|
+
for inference tasks.
|
|
522
|
+
|
|
523
|
+
Examples
|
|
524
|
+
--------
|
|
525
|
+
>>> # Assuming model is already trained
|
|
526
|
+
>>> X_test = np.array([[1.0, 2.0], [3.0, 4.0]])
|
|
527
|
+
>>> predictions = model.predict_model(X_test)
|
|
528
|
+
>>> print(predictions)
|
|
529
|
+
"""
|
|
261
530
|
return self.model(X, training=False)
|
|
262
531
|
|
|
263
|
-
def evaluate(self):
|
|
532
|
+
def evaluate(self) -> tuple[list, list, float, float]:
|
|
533
|
+
"""
|
|
534
|
+
Evaluate the model performance on validation data and compute loss metrics.
|
|
535
|
+
|
|
536
|
+
This method performs model evaluation by computing prediction errors and
|
|
537
|
+
saving training/validation loss curves. It calculates both prediction error
|
|
538
|
+
(difference between predicted and actual values) and raw error (difference
|
|
539
|
+
between input and actual values). The method also generates and saves a
|
|
540
|
+
plot of the training and validation loss over epochs.
|
|
541
|
+
|
|
542
|
+
Parameters
|
|
543
|
+
----------
|
|
544
|
+
self : object
|
|
545
|
+
The instance of the class containing the model and data attributes.
|
|
546
|
+
|
|
547
|
+
Returns
|
|
548
|
+
-------
|
|
549
|
+
tuple[list, list, float, float]
|
|
550
|
+
A tuple containing:
|
|
551
|
+
- training_loss : list
|
|
552
|
+
List of training loss values per epoch
|
|
553
|
+
- validation_loss : list
|
|
554
|
+
List of validation loss values per epoch
|
|
555
|
+
- prediction_error : float
|
|
556
|
+
Mean squared error between predicted and actual values
|
|
557
|
+
- raw_error : float
|
|
558
|
+
Mean squared error between input features and actual values
|
|
559
|
+
|
|
560
|
+
Notes
|
|
561
|
+
-----
|
|
562
|
+
This method modifies the instance attributes:
|
|
563
|
+
- self.lossfilename: Path to the saved loss plot
|
|
564
|
+
- self.pred_error: Computed prediction error
|
|
565
|
+
- self.raw_error: Computed raw error
|
|
566
|
+
- self.loss: Training loss history
|
|
567
|
+
- self.val_loss: Validation loss history
|
|
568
|
+
|
|
569
|
+
The method saves:
|
|
570
|
+
- Loss plot as PNG file
|
|
571
|
+
- Loss metrics as text file
|
|
572
|
+
|
|
573
|
+
Examples
|
|
574
|
+
--------
|
|
575
|
+
>>> model = MyModel()
|
|
576
|
+
>>> train_loss, val_loss, pred_error, raw_error = model.evaluate()
|
|
577
|
+
>>> print(f"Prediction Error: {pred_error}")
|
|
578
|
+
Prediction Error: 0.1234
|
|
579
|
+
"""
|
|
264
580
|
self.lossfilename = os.path.join(self.modelname, "loss.png")
|
|
265
581
|
LGR.info(f"lossfilename: {self.lossfilename}")
|
|
266
582
|
|
|
@@ -301,7 +617,44 @@ class DeepLearningFilter:
|
|
|
301
617
|
|
|
302
618
|
return self.loss, self.val_loss, self.pred_error, self.raw_error
|
|
303
619
|
|
|
304
|
-
def initmetadata(self):
|
|
620
|
+
def initmetadata(self) -> None:
|
|
621
|
+
"""
|
|
622
|
+
Initialize and store metadata information for the model.
|
|
623
|
+
|
|
624
|
+
This function creates a dictionary containing various model configuration parameters
|
|
625
|
+
and writes them to a JSON file for future reference and reproducibility.
|
|
626
|
+
|
|
627
|
+
Parameters
|
|
628
|
+
----------
|
|
629
|
+
self : object
|
|
630
|
+
The instance of the class containing the metadata attributes.
|
|
631
|
+
|
|
632
|
+
Returns
|
|
633
|
+
-------
|
|
634
|
+
None
|
|
635
|
+
This function does not return any value but writes metadata to a JSON file.
|
|
636
|
+
|
|
637
|
+
Notes
|
|
638
|
+
-----
|
|
639
|
+
The metadata includes:
|
|
640
|
+
- Window size for processing
|
|
641
|
+
- Bad point handling flag
|
|
642
|
+
- FFT usage flag
|
|
643
|
+
- Exclusion threshold
|
|
644
|
+
- Number of epochs and layers
|
|
645
|
+
- Dropout rate
|
|
646
|
+
- Operating system platform
|
|
647
|
+
- Model name
|
|
648
|
+
|
|
649
|
+
The metadata is saved to ``{modelname}/model_meta.json`` where ``modelname``
|
|
650
|
+
is the model's name attribute.
|
|
651
|
+
|
|
652
|
+
Examples
|
|
653
|
+
--------
|
|
654
|
+
>>> model = MyModel()
|
|
655
|
+
>>> model.initmetadata()
|
|
656
|
+
>>> # Metadata stored in modelname/model_meta.json
|
|
657
|
+
"""
|
|
305
658
|
self.infodict = {}
|
|
306
659
|
self.infodict["window_size"] = self.window_size
|
|
307
660
|
self.infodict["usebadpts"] = self.usebadpts
|
|
@@ -314,14 +667,87 @@ class DeepLearningFilter:
|
|
|
314
667
|
self.infodict["modelname"] = self.modelname
|
|
315
668
|
tide_io.writedicttojson(self.infodict, os.path.join(self.modelname, "model_meta.json"))
|
|
316
669
|
|
|
317
|
-
def updatemetadata(self):
|
|
670
|
+
def updatemetadata(self) -> None:
|
|
671
|
+
"""
|
|
672
|
+
Update metadata dictionary with model metrics and save to JSON file.
|
|
673
|
+
|
|
674
|
+
This method updates the internal information dictionary with various model
|
|
675
|
+
performance metrics and writes the complete metadata to a JSON file for
|
|
676
|
+
model persistence and tracking.
|
|
677
|
+
|
|
678
|
+
Parameters
|
|
679
|
+
----------
|
|
680
|
+
self : object
|
|
681
|
+
The instance of the class containing the metadata and model information.
|
|
682
|
+
Expected to have the following attributes:
|
|
683
|
+
- infodict : dict
|
|
684
|
+
Dictionary containing model metadata.
|
|
685
|
+
- loss : float
|
|
686
|
+
Training loss value.
|
|
687
|
+
- val_loss : float
|
|
688
|
+
Validation loss value.
|
|
689
|
+
- raw_error : float
|
|
690
|
+
Raw error metric.
|
|
691
|
+
- pred_error : float
|
|
692
|
+
Prediction error metric.
|
|
693
|
+
- modelname : str
|
|
694
|
+
Name/path of the model for file output.
|
|
695
|
+
|
|
696
|
+
Returns
|
|
697
|
+
-------
|
|
698
|
+
None
|
|
699
|
+
This method does not return any value but modifies the `infodict` in-place
|
|
700
|
+
and writes to a JSON file.
|
|
701
|
+
|
|
702
|
+
Notes
|
|
703
|
+
-----
|
|
704
|
+
The method writes metadata to ``{modelname}/model_meta.json`` where
|
|
705
|
+
``modelname`` is the model name attribute of the instance.
|
|
706
|
+
|
|
707
|
+
Examples
|
|
708
|
+
--------
|
|
709
|
+
>>> model = MyModel()
|
|
710
|
+
>>> model.updatemetadata()
|
|
711
|
+
>>> # Creates model_meta.json with loss, val_loss, raw_error, and pred_error
|
|
712
|
+
"""
|
|
318
713
|
self.infodict["loss"] = self.loss
|
|
319
714
|
self.infodict["val_loss"] = self.val_loss
|
|
320
715
|
self.infodict["raw_error"] = self.raw_error
|
|
321
716
|
self.infodict["prediction_error"] = self.pred_error
|
|
322
717
|
tide_io.writedicttojson(self.infodict, os.path.join(self.modelname, "model_meta.json"))
|
|
323
718
|
|
|
324
|
-
def savemodel(self, altname=None):
|
|
719
|
+
def savemodel(self, altname: str | None = None) -> None:
|
|
720
|
+
"""
|
|
721
|
+
Save the model to disk with the specified name.
|
|
722
|
+
|
|
723
|
+
This method saves the current model to a Keras file format (.keras) in a
|
|
724
|
+
directory named according to the model name or an alternative name provided.
|
|
725
|
+
|
|
726
|
+
Parameters
|
|
727
|
+
----------
|
|
728
|
+
altname : str, optional
|
|
729
|
+
Alternative name to use for saving the model. If None, uses the
|
|
730
|
+
model's default name stored in `self.modelname`. Default is None.
|
|
731
|
+
|
|
732
|
+
Returns
|
|
733
|
+
-------
|
|
734
|
+
None
|
|
735
|
+
This method does not return any value.
|
|
736
|
+
|
|
737
|
+
Notes
|
|
738
|
+
-----
|
|
739
|
+
The model is saved in the Keras format (.keras) and stored in a directory
|
|
740
|
+
with the same name as the model. The method logs the saving operation
|
|
741
|
+
using the logger instance `LGR`.
|
|
742
|
+
|
|
743
|
+
Examples
|
|
744
|
+
--------
|
|
745
|
+
>>> # Save model with default name
|
|
746
|
+
>>> savemodel()
|
|
747
|
+
>>>
|
|
748
|
+
>>> # Save model with alternative name
|
|
749
|
+
>>> savemodel(altname="my_custom_model")
|
|
750
|
+
"""
|
|
325
751
|
if altname is None:
|
|
326
752
|
modelsavename = self.modelname
|
|
327
753
|
else:
|
|
@@ -329,7 +755,51 @@ class DeepLearningFilter:
|
|
|
329
755
|
LGR.info(f"saving {modelsavename}")
|
|
330
756
|
self.model.save(os.path.join(modelsavename, "model.keras"))
|
|
331
757
|
|
|
332
|
-
def loadmodel(self, modelname, verbose=False):
|
|
758
|
+
def loadmodel(self, modelname: str, verbose: bool = False) -> None:
|
|
759
|
+
"""
|
|
760
|
+
Load a trained model from disk and initialize model parameters.
|
|
761
|
+
|
|
762
|
+
Load a Keras model from the specified model directory, along with associated
|
|
763
|
+
metadata and configuration information. The function attempts to load the model
|
|
764
|
+
in Keras format first, falling back to HDF5 format if the Keras format is not found.
|
|
765
|
+
|
|
766
|
+
Parameters
|
|
767
|
+
----------
|
|
768
|
+
modelname : str
|
|
769
|
+
Name of the model to load, corresponding to a subdirectory in ``self.modelpath``.
|
|
770
|
+
verbose : bool, optional
|
|
771
|
+
If True, print model summary and metadata information. Default is False.
|
|
772
|
+
|
|
773
|
+
Returns
|
|
774
|
+
-------
|
|
775
|
+
None
|
|
776
|
+
This method modifies the instance attributes in-place and does not return anything.
|
|
777
|
+
|
|
778
|
+
Notes
|
|
779
|
+
-----
|
|
780
|
+
The function attempts to load the model in the following order:
|
|
781
|
+
1. Keras format (``model.keras``)
|
|
782
|
+
2. HDF5 format (``model.h5``)
|
|
783
|
+
|
|
784
|
+
If neither format is found, the function exits with an error message.
|
|
785
|
+
|
|
786
|
+
The loaded model metadata is stored in ``self.infodict``, and model configuration
|
|
787
|
+
is stored in ``self.config``. Additional attributes like ``window_size`` and ``usebadpts``
|
|
788
|
+
are extracted from the metadata and stored as instance attributes.
|
|
789
|
+
|
|
790
|
+
Examples
|
|
791
|
+
--------
|
|
792
|
+
>>> loader = ModelLoader()
|
|
793
|
+
>>> loader.loadmodel("my_model", verbose=True)
|
|
794
|
+
loading my_model
|
|
795
|
+
Model: "sequential"
|
|
796
|
+
_________________________________________________________________
|
|
797
|
+
Layer (type) Output Shape Param #
|
|
798
|
+
=================================================================
|
|
799
|
+
...
|
|
800
|
+
>>> print(loader.window_size)
|
|
801
|
+
100
|
|
802
|
+
"""
|
|
333
803
|
# read in the data
|
|
334
804
|
LGR.info(f"loading {modelname}")
|
|
335
805
|
try:
|
|
@@ -361,7 +831,41 @@ class DeepLearningFilter:
|
|
|
361
831
|
self.trained = True
|
|
362
832
|
LGR.info(f"{modelname} loaded")
|
|
363
833
|
|
|
364
|
-
def initialize(self):
|
|
834
|
+
def initialize(self) -> None:
|
|
835
|
+
"""
|
|
836
|
+
Initialize the model by setting up network architecture and metadata.
|
|
837
|
+
|
|
838
|
+
This method performs a series of initialization steps including retrieving
|
|
839
|
+
the model name, creating the network architecture, displaying model summary,
|
|
840
|
+
saving the model configuration, initializing metadata, and setting appropriate
|
|
841
|
+
flags to indicate initialization status.
|
|
842
|
+
|
|
843
|
+
Parameters
|
|
844
|
+
----------
|
|
845
|
+
self : object
|
|
846
|
+
The instance of the model class being initialized.
|
|
847
|
+
|
|
848
|
+
Returns
|
|
849
|
+
-------
|
|
850
|
+
None
|
|
851
|
+
This method does not return any value.
|
|
852
|
+
|
|
853
|
+
Notes
|
|
854
|
+
-----
|
|
855
|
+
This method should be called before any training or prediction operations.
|
|
856
|
+
The initialization process sets `self.initialized` to True and `self.trained`
|
|
857
|
+
to False, indicating that the model is ready for training but has not been
|
|
858
|
+
trained yet.
|
|
859
|
+
|
|
860
|
+
Examples
|
|
861
|
+
--------
|
|
862
|
+
>>> model = MyModel()
|
|
863
|
+
>>> model.initialize()
|
|
864
|
+
>>> print(model.initialized)
|
|
865
|
+
True
|
|
866
|
+
>>> print(model.trained)
|
|
867
|
+
False
|
|
868
|
+
"""
|
|
365
869
|
self.getname()
|
|
366
870
|
self.makenet()
|
|
367
871
|
self.model.summary()
|
|
@@ -370,7 +874,47 @@ class DeepLearningFilter:
|
|
|
370
874
|
self.initialized = True
|
|
371
875
|
self.trained = False
|
|
372
876
|
|
|
373
|
-
def train(self):
|
|
877
|
+
def train(self) -> None:
|
|
878
|
+
"""
|
|
879
|
+
Train the model using the provided training and validation datasets.
|
|
880
|
+
|
|
881
|
+
This method performs model training with optional pretraining and logging. It supports
|
|
882
|
+
TensorBoard logging, model checkpointing, early stopping, and NaN termination. The trained
|
|
883
|
+
model is saved at the end of training.
|
|
884
|
+
|
|
885
|
+
Parameters
|
|
886
|
+
----------
|
|
887
|
+
self : object
|
|
888
|
+
The instance of the class containing the model and training configuration.
|
|
889
|
+
Expected attributes include:
|
|
890
|
+
- `model`: The Keras model to be trained.
|
|
891
|
+
- `train_x`, `train_y`: Training data inputs and labels.
|
|
892
|
+
- `val_x`, `val_y`: Validation data inputs and labels.
|
|
893
|
+
- `modelname`: Name of the model for saving purposes.
|
|
894
|
+
- `usetensorboard`: Boolean flag to enable TensorBoard logging.
|
|
895
|
+
- `num_pretrain_epochs`: Number of epochs for pretraining phase.
|
|
896
|
+
- `num_epochs`: Total number of training epochs.
|
|
897
|
+
- `savemodel()`: Method to save the trained model.
|
|
898
|
+
|
|
899
|
+
Returns
|
|
900
|
+
-------
|
|
901
|
+
None
|
|
902
|
+
This function does not return any value.
|
|
903
|
+
|
|
904
|
+
Notes
|
|
905
|
+
-----
|
|
906
|
+
- If `self.usetensorboard` is True, TensorBoard logging is enabled.
|
|
907
|
+
- If `self.num_pretrain_epochs` is greater than 0, a pretraining phase is performed
|
|
908
|
+
before the main training loop.
|
|
909
|
+
- The model is saved after training using the `savemodel()` method.
|
|
910
|
+
- Training uses `ModelCheckpoint`, `EarlyStopping`, and `TerminateOnNaN` callbacks
|
|
911
|
+
to manage training process and prevent overfitting or NaN issues.
|
|
912
|
+
|
|
913
|
+
Examples
|
|
914
|
+
--------
|
|
915
|
+
>>> trainer = ModelTrainer(model, train_x, train_y, val_x, val_y)
|
|
916
|
+
>>> trainer.train()
|
|
917
|
+
"""
|
|
374
918
|
self.intermediatemodelpath = os.path.join(
|
|
375
919
|
self.modelname, "model_e{epoch:02d}_v{val_loss:.4f}.keras"
|
|
376
920
|
)
|
|
@@ -421,7 +965,43 @@ class DeepLearningFilter:
|
|
|
421
965
|
self.savemodel()
|
|
422
966
|
self.trained = True
|
|
423
967
|
|
|
424
|
-
def apply(self, inputdata, badpts=None):
|
|
968
|
+
def apply(self, inputdata: NDArray, badpts: NDArray | None = None) -> NDArray:
|
|
969
|
+
"""
|
|
970
|
+
Apply a sliding-window prediction model to the input data, optionally incorporating bad points.
|
|
971
|
+
|
|
972
|
+
This function performs a sliding-window prediction using a pre-trained model. It scales the input
|
|
973
|
+
data using the median absolute deviation (MAD), applies the model to overlapping windows of data,
|
|
974
|
+
and aggregates predictions with a weighted scheme. Optionally, bad points can be included in
|
|
975
|
+
the prediction process to influence the model's behavior.
|
|
976
|
+
|
|
977
|
+
Parameters
|
|
978
|
+
----------
|
|
979
|
+
inputdata : NDArray
|
|
980
|
+
Input data array of shape (N,) to be processed.
|
|
981
|
+
badpts : NDArray | None, optional
|
|
982
|
+
Array of same shape as `inputdata` indicating bad or invalid points. If None, no bad points
|
|
983
|
+
are considered. Default is None.
|
|
984
|
+
|
|
985
|
+
Returns
|
|
986
|
+
-------
|
|
987
|
+
NDArray
|
|
988
|
+
Predicted data array of the same shape as `inputdata`, with predictions aggregated and
|
|
989
|
+
weighted across overlapping windows.
|
|
990
|
+
|
|
991
|
+
Notes
|
|
992
|
+
-----
|
|
993
|
+
- The function uses a sliding window of size `self.window_size` to process input data.
|
|
994
|
+
- Predictions are aggregated by summing over overlapping windows.
|
|
995
|
+
- A triangular weight scheme is applied to the aggregated predictions to reduce edge effects.
|
|
996
|
+
- If `self.usebadpts` is True, `badpts` are included as an additional feature in the model input.
|
|
997
|
+
|
|
998
|
+
Examples
|
|
999
|
+
--------
|
|
1000
|
+
>>> model = MyModel(window_size=10, usebadpts=True)
|
|
1001
|
+
>>> input_data = np.random.randn(100)
|
|
1002
|
+
>>> bad_points = np.zeros_like(input_data)
|
|
1003
|
+
>>> result = model.apply(input_data, bad_points)
|
|
1004
|
+
"""
|
|
425
1005
|
initscale = mad(inputdata)
|
|
426
1006
|
scaleddata = inputdata / initscale
|
|
427
1007
|
predicteddata = scaleddata * 0.0
|
|
@@ -460,14 +1040,104 @@ class MultiscaleCNNDLFilter(DeepLearningFilter):
|
|
|
460
1040
|
# it takes a time series as an input, performs 1-D convolution, and returns it as an output ready for concatenation
|
|
461
1041
|
def __init__(
|
|
462
1042
|
self,
|
|
463
|
-
num_filters=10,
|
|
464
|
-
kernel_sizes=[4, 8, 12],
|
|
465
|
-
input_lens=[64, 128, 192],
|
|
466
|
-
input_width=1,
|
|
467
|
-
dilation_rate=1,
|
|
1043
|
+
num_filters: int = 10,
|
|
1044
|
+
kernel_sizes: list[int] = [4, 8, 12],
|
|
1045
|
+
input_lens: list[int] = [64, 128, 192],
|
|
1046
|
+
input_width: int = 1,
|
|
1047
|
+
dilation_rate: int = 1,
|
|
468
1048
|
*args,
|
|
469
1049
|
**kwargs,
|
|
470
|
-
):
|
|
1050
|
+
) -> None:
|
|
1051
|
+
"""
|
|
1052
|
+
Initialize the MultiscaleCNNDLFilter.
|
|
1053
|
+
|
|
1054
|
+
This constructor initializes a multiscale CNN filter with specified parameters
|
|
1055
|
+
for processing sequential data with multiple kernel sizes and dilation rates.
|
|
1056
|
+
|
|
1057
|
+
Parameters
|
|
1058
|
+
----------
|
|
1059
|
+
num_filters : int, optional
|
|
1060
|
+
Number of filters to use in the convolutional layers. Default is 10.
|
|
1061
|
+
kernel_sizes : list of int, optional
|
|
1062
|
+
List of kernel sizes to use for different convolutional layers.
|
|
1063
|
+
Default is [4, 8, 12].
|
|
1064
|
+
input_lens : list of int, optional
|
|
1065
|
+
List of input sequence lengths to process. Default is [64, 128, 192].
|
|
1066
|
+
input_width : int, optional
|
|
1067
|
+
Width of the input data (number of input channels). Default is 1.
|
|
1068
|
+
dilation_rate : int, optional
|
|
1069
|
+
Dilation rate for the convolutional layers. Default is 1.
|
|
1070
|
+
*args
|
|
1071
|
+
Variable length argument list passed to parent class.
|
|
1072
|
+
**kwargs
|
|
1073
|
+
Arbitrary keyword arguments passed to parent class.
|
|
1074
|
+
|
|
1075
|
+
Returns
|
|
1076
|
+
-------
|
|
1077
|
+
None
|
|
1078
|
+
This method initializes the object and does not return any value.
|
|
1079
|
+
|
|
1080
|
+
Notes
|
|
1081
|
+
-----
|
|
1082
|
+
The initialized object will store network configuration information in
|
|
1083
|
+
`infodict` including network type, number of filters, kernel sizes, input lengths,
|
|
1084
|
+
and input width. The parent class initialization is called using `super()`.
|
|
1085
|
+
|
|
1086
|
+
Examples
|
|
1087
|
+
--------
|
|
1088
|
+
>>> filter = MultiscaleCNNDLFilter(
|
|
1089
|
+
... num_filters=20,
|
|
1090
|
+
... kernel_sizes=[3, 6, 9],
|
|
1091
|
+
... input_lens=[32, 64, 128],
|
|
1092
|
+
... input_width=2,
|
|
1093
|
+
... dilation_rate=2
|
|
1094
|
+
... )
|
|
1095
|
+
"""
|
|
1096
|
+
"""
|
|
1097
|
+
Initialize the MultiscaleCNNDLFilter.
|
|
1098
|
+
|
|
1099
|
+
This constructor initializes a multiscale CNN filter with specified parameters
|
|
1100
|
+
for processing sequential data with multiple kernel sizes and dilation rates.
|
|
1101
|
+
|
|
1102
|
+
Parameters
|
|
1103
|
+
----------
|
|
1104
|
+
num_filters : int, optional
|
|
1105
|
+
Number of filters to use in the convolutional layers, default is 10
|
|
1106
|
+
kernel_sizes : list of int, optional
|
|
1107
|
+
List of kernel sizes to use for different convolutional layers,
|
|
1108
|
+
default is [4, 8, 12]
|
|
1109
|
+
input_lens : list of int, optional
|
|
1110
|
+
List of input sequence lengths to process, default is [64, 128, 192]
|
|
1111
|
+
input_width : int, optional
|
|
1112
|
+
Width of the input data (number of input channels), default is 1
|
|
1113
|
+
dilation_rate : int, optional
|
|
1114
|
+
Dilation rate for the convolutional layers, default is 1
|
|
1115
|
+
*args
|
|
1116
|
+
Variable length argument list passed to parent class
|
|
1117
|
+
**kwargs
|
|
1118
|
+
Arbitrary keyword arguments passed to parent class
|
|
1119
|
+
|
|
1120
|
+
Returns
|
|
1121
|
+
-------
|
|
1122
|
+
None
|
|
1123
|
+
This method initializes the object and does not return any value
|
|
1124
|
+
|
|
1125
|
+
Notes
|
|
1126
|
+
-----
|
|
1127
|
+
The initialized object will store network configuration information in
|
|
1128
|
+
`infodict` including network type, number of filters, kernel sizes, input lengths,
|
|
1129
|
+
and input width. The parent class initialization is called using super().
|
|
1130
|
+
|
|
1131
|
+
Examples
|
|
1132
|
+
--------
|
|
1133
|
+
>>> filter = MultiscaleCNNDLFilter(
|
|
1134
|
+
... num_filters=20,
|
|
1135
|
+
... kernel_sizes=[3, 6, 9],
|
|
1136
|
+
... input_lens=[32, 64, 128],
|
|
1137
|
+
... input_width=2,
|
|
1138
|
+
... dilation_rate=2
|
|
1139
|
+
... )
|
|
1140
|
+
"""
|
|
471
1141
|
self.num_filters = num_filters
|
|
472
1142
|
self.kernel_sizes = kernel_sizes
|
|
473
1143
|
self.input_lens = input_lens
|
|
@@ -481,10 +1151,74 @@ class MultiscaleCNNDLFilter(DeepLearningFilter):
|
|
|
481
1151
|
super(MultiscaleCNNDLFilter, self).__init__(*args, **kwargs)
|
|
482
1152
|
|
|
483
1153
|
def getname(self):
|
|
1154
|
+
"""
|
|
1155
|
+
Generate and return model name based on configuration parameters.
|
|
1156
|
+
|
|
1157
|
+
This method constructs a descriptive model name by joining various configuration
|
|
1158
|
+
parameters with specific prefixes and zero-padded numbers. The generated name
|
|
1159
|
+
is used to create a unique model identifier and corresponding directory path.
|
|
1160
|
+
|
|
1161
|
+
Parameters
|
|
1162
|
+
----------
|
|
1163
|
+
self : object
|
|
1164
|
+
The instance of the class containing the following attributes:
|
|
1165
|
+
|
|
1166
|
+
- window_size : int
|
|
1167
|
+
Size of the input window
|
|
1168
|
+
- num_layers : int
|
|
1169
|
+
Number of layers in the model
|
|
1170
|
+
- num_filters : int
|
|
1171
|
+
Number of filters in the model
|
|
1172
|
+
- kernel_size : int
|
|
1173
|
+
Size of the convolutional kernel
|
|
1174
|
+
- num_epochs : int
|
|
1175
|
+
Number of training epochs
|
|
1176
|
+
- excludethresh : float
|
|
1177
|
+
Threshold for excluding data points
|
|
1178
|
+
- corrthresh : float
|
|
1179
|
+
Correlation threshold for filtering
|
|
1180
|
+
- step : int
|
|
1181
|
+
Step size for processing
|
|
1182
|
+
- dilation_rate : int
|
|
1183
|
+
Dilation rate for convolutional layers
|
|
1184
|
+
- activation : str
|
|
1185
|
+
Activation function name
|
|
1186
|
+
- usebadpts : bool
|
|
1187
|
+
Whether to use bad points in training
|
|
1188
|
+
- excludebysubject : bool
|
|
1189
|
+
Whether to exclude data by subject
|
|
1190
|
+
- namesuffix : str, optional
|
|
1191
|
+
Additional suffix to append to the model name
|
|
1192
|
+
- modelroot : str
|
|
1193
|
+
Root directory for model storage
|
|
1194
|
+
|
|
1195
|
+
Returns
|
|
1196
|
+
-------
|
|
1197
|
+
None
|
|
1198
|
+
This method modifies the instance attributes `modelname` and `modelpath`
|
|
1199
|
+
but does not return any value.
|
|
1200
|
+
|
|
1201
|
+
Notes
|
|
1202
|
+
-----
|
|
1203
|
+
The generated model name follows this format:
|
|
1204
|
+
"model_multiscalecnn_tf2_wXxx_lYy_fnZz_flZz_eXxx_tY_ctZ_sZ_dZ_activation[_usebadpts][_excludebysubject][_suffix]"
|
|
1205
|
+
|
|
1206
|
+
Where Xxx, Yy, Zz, etc. represent zero-padded numbers based on the parameter values.
|
|
1207
|
+
|
|
1208
|
+
Examples
|
|
1209
|
+
--------
|
|
1210
|
+
>>> model = MyModel()
|
|
1211
|
+
>>> model.window_size = 100
|
|
1212
|
+
>>> model.num_layers = 5
|
|
1213
|
+
>>> model.getname()
|
|
1214
|
+
>>> print(model.modelname)
|
|
1215
|
+
'model_multiscalecnn_tf2_w100_l05_fn05_fl05_e001_t0.5_ct0.8_s1_d1_relu'
|
|
1216
|
+
"""
|
|
484
1217
|
self.modelname = "_".join(
|
|
485
1218
|
[
|
|
486
1219
|
"model",
|
|
487
1220
|
"multiscalecnn",
|
|
1221
|
+
"tf2",
|
|
488
1222
|
"w" + str(self.window_size).zfill(3),
|
|
489
1223
|
"l" + str(self.num_layers).zfill(2),
|
|
490
1224
|
"fn" + str(self.num_filters).zfill(2),
|
|
@@ -511,6 +1245,41 @@ class MultiscaleCNNDLFilter(DeepLearningFilter):
|
|
|
511
1245
|
pass
|
|
512
1246
|
|
|
513
1247
|
def makesubnet(self, inputlen, kernelsize):
|
|
1248
|
+
"""
|
|
1249
|
+
Create a 1D convolutional neural network submodel for time series processing.
|
|
1250
|
+
|
|
1251
|
+
This function constructs a neural network submodel that processes time series data
|
|
1252
|
+
through 1D convolutional layers followed by global max pooling and dense layers
|
|
1253
|
+
with dropout regularization. The model is designed for feature extraction from
|
|
1254
|
+
sequential data.
|
|
1255
|
+
|
|
1256
|
+
Parameters
|
|
1257
|
+
----------
|
|
1258
|
+
inputlen : int
|
|
1259
|
+
Length of the input time series sequence.
|
|
1260
|
+
kernelsize : int
|
|
1261
|
+
Size of the convolutional kernel for 1D convolution operation.
|
|
1262
|
+
|
|
1263
|
+
Returns
|
|
1264
|
+
-------
|
|
1265
|
+
Model
|
|
1266
|
+
Keras Model object representing the constructed submodel with the following structure:
|
|
1267
|
+
Input -> Conv1D -> GlobalMaxPool1D -> Dense -> Dropout -> Dense -> Output
|
|
1268
|
+
|
|
1269
|
+
Notes
|
|
1270
|
+
-----
|
|
1271
|
+
The model architecture includes:
|
|
1272
|
+
- 1D convolution with tanh activation
|
|
1273
|
+
- Global max pooling for sequence reduction
|
|
1274
|
+
- Two dense layers with tanh activation
|
|
1275
|
+
- Dropout regularization (0.3) after first dense layer
|
|
1276
|
+
- Uses 'same' padding for convolutional layer
|
|
1277
|
+
|
|
1278
|
+
Examples
|
|
1279
|
+
--------
|
|
1280
|
+
>>> model = makesubnet(inputlen=100, kernelsize=3)
|
|
1281
|
+
>>> model.summary()
|
|
1282
|
+
"""
|
|
514
1283
|
# the input is a time series of length input_len and width input_width
|
|
515
1284
|
input_seq = Input(shape=(inputlen, self.input_width))
|
|
516
1285
|
|
|
@@ -527,6 +1296,48 @@ class MultiscaleCNNDLFilter(DeepLearningFilter):
|
|
|
527
1296
|
return basemodel
|
|
528
1297
|
|
|
529
1298
|
def makenet(self):
|
|
1299
|
+
"""
|
|
1300
|
+
Create a multi-scale neural network for time series analysis.
|
|
1301
|
+
|
|
1302
|
+
This function constructs a neural network model that processes time series data at
|
|
1303
|
+
multiple resolutions (original, medium, and small scale). The model uses separate
|
|
1304
|
+
sub-networks for each resolution level and concatenates their outputs to make
|
|
1305
|
+
predictions. The architecture leverages different kernel sizes for different
|
|
1306
|
+
down-sampled versions to capture features at multiple temporal scales.
|
|
1307
|
+
|
|
1308
|
+
Parameters
|
|
1309
|
+
----------
|
|
1310
|
+
self : object
|
|
1311
|
+
The instance containing the following attributes:
|
|
1312
|
+
- inputs_lens : list of int
|
|
1313
|
+
Lengths of input sequences for small, medium, and original scales.
|
|
1314
|
+
- input_width : int
|
|
1315
|
+
Width of input features.
|
|
1316
|
+
- kernel_sizes : list of int
|
|
1317
|
+
Kernel sizes for convolutional layers corresponding to each scale.
|
|
1318
|
+
- makesubnet : callable
|
|
1319
|
+
Function to create sub-network for a given sequence length and kernel size.
|
|
1320
|
+
|
|
1321
|
+
Returns
|
|
1322
|
+
-------
|
|
1323
|
+
None
|
|
1324
|
+
This method modifies the instance in-place by setting the `model` attribute
|
|
1325
|
+
to the constructed Keras model.
|
|
1326
|
+
|
|
1327
|
+
Notes
|
|
1328
|
+
-----
|
|
1329
|
+
The model architecture:
|
|
1330
|
+
1. Takes three inputs at different resolutions.
|
|
1331
|
+
2. Processes each input through separate sub-networks with scale-specific kernels.
|
|
1332
|
+
3. Concatenates the embeddings from all branches.
|
|
1333
|
+
4. Applies a final dense layer with sigmoid activation for binary classification.
|
|
1334
|
+
|
|
1335
|
+
Examples
|
|
1336
|
+
--------
|
|
1337
|
+
>>> # Assuming self is an instance with proper attributes set
|
|
1338
|
+
>>> self.makenet()
|
|
1339
|
+
>>> print(self.model.summary())
|
|
1340
|
+
"""
|
|
530
1341
|
# the inputs to the branches are the original time series, and its down-sampled versions
|
|
531
1342
|
input_smallseq = Input(shape=(self.inputs_lens[0], self.input_width))
|
|
532
1343
|
input_medseq = Input(shape=(self.inputs_lens[1], self.input_width))
|
|
@@ -548,7 +1359,48 @@ class MultiscaleCNNDLFilter(DeepLearningFilter):
|
|
|
548
1359
|
|
|
549
1360
|
|
|
550
1361
|
class CNNDLFilter(DeepLearningFilter):
|
|
551
|
-
def __init__(
|
|
1362
|
+
def __init__(
|
|
1363
|
+
self,
|
|
1364
|
+
num_filters: int = 10,
|
|
1365
|
+
kernel_size: int = 5,
|
|
1366
|
+
dilation_rate: int = 1,
|
|
1367
|
+
*args,
|
|
1368
|
+
**kwargs,
|
|
1369
|
+
) -> None:
|
|
1370
|
+
"""
|
|
1371
|
+
Initialize the CNNDLFilter layer.
|
|
1372
|
+
|
|
1373
|
+
Parameters
|
|
1374
|
+
----------
|
|
1375
|
+
num_filters : int, optional
|
|
1376
|
+
Number of convolutional filters to use, by default 10.
|
|
1377
|
+
kernel_size : int, optional
|
|
1378
|
+
Size of the convolutional kernel, by default 5.
|
|
1379
|
+
dilation_rate : int, optional
|
|
1380
|
+
Dilation rate for the convolutional layer, by default 1.
|
|
1381
|
+
*args
|
|
1382
|
+
Variable length argument list passed to parent class.
|
|
1383
|
+
**kwargs
|
|
1384
|
+
Arbitrary keyword arguments passed to parent class.
|
|
1385
|
+
|
|
1386
|
+
Returns
|
|
1387
|
+
-------
|
|
1388
|
+
None
|
|
1389
|
+
This method initializes the layer and does not return any value.
|
|
1390
|
+
|
|
1391
|
+
Notes
|
|
1392
|
+
-----
|
|
1393
|
+
This constructor sets up a convolutional layer with specified parameters
|
|
1394
|
+
and registers the network type as "cnn" in the infodict. The dilation rate
|
|
1395
|
+
controls the spacing between kernel points, allowing for wider receptive
|
|
1396
|
+
fields without increasing the number of parameters.
|
|
1397
|
+
|
|
1398
|
+
Examples
|
|
1399
|
+
--------
|
|
1400
|
+
>>> layer = CNNDLFilter(num_filters=32, kernel_size=3, dilation_rate=2)
|
|
1401
|
+
>>> print(layer.num_filters)
|
|
1402
|
+
32
|
|
1403
|
+
"""
|
|
552
1404
|
self.num_filters = num_filters
|
|
553
1405
|
self.kernel_size = kernel_size
|
|
554
1406
|
self.dilation_rate = dilation_rate
|
|
@@ -558,10 +1410,79 @@ class CNNDLFilter(DeepLearningFilter):
|
|
|
558
1410
|
super(CNNDLFilter, self).__init__(*args, **kwargs)
|
|
559
1411
|
|
|
560
1412
|
def getname(self):
|
|
1413
|
+
"""
|
|
1414
|
+
Generate and return the model name based on configuration parameters.
|
|
1415
|
+
|
|
1416
|
+
This method constructs a descriptive model name by joining various configuration
|
|
1417
|
+
parameters with specific prefixes and zero-padded numbers. The resulting name
|
|
1418
|
+
is used to create a unique directory path for model storage.
|
|
1419
|
+
|
|
1420
|
+
Parameters
|
|
1421
|
+
----------
|
|
1422
|
+
self : object
|
|
1423
|
+
The instance of the class containing model configuration parameters.
|
|
1424
|
+
Expected attributes include:
|
|
1425
|
+
- window_size : int
|
|
1426
|
+
- num_layers : int
|
|
1427
|
+
- num_filters : int
|
|
1428
|
+
- kernel_size : int
|
|
1429
|
+
- num_epochs : int
|
|
1430
|
+
- excludethresh : float
|
|
1431
|
+
- corrthresh : float
|
|
1432
|
+
- step : int
|
|
1433
|
+
- dilation_rate : int
|
|
1434
|
+
- activation : str
|
|
1435
|
+
- modelroot : str
|
|
1436
|
+
- usebadpts : bool
|
|
1437
|
+
- excludebysubject : bool
|
|
1438
|
+
- namesuffix : str, optional
|
|
1439
|
+
|
|
1440
|
+
Returns
|
|
1441
|
+
-------
|
|
1442
|
+
None
|
|
1443
|
+
This method does not return a value but sets the following attributes:
|
|
1444
|
+
- `self.modelname`: str, the constructed model name
|
|
1445
|
+
- `self.modelpath`: str, the full path to the model directory
|
|
1446
|
+
|
|
1447
|
+
Notes
|
|
1448
|
+
-----
|
|
1449
|
+
The generated model name follows this format:
|
|
1450
|
+
"model_cnn_tf2_wXXX_lYY_fnZZ_flZZ_eXXX_tY_ctZ_sZ_dZ_activation[_usebadpts][_excludebysubject][_suffix]"
|
|
1451
|
+
|
|
1452
|
+
Where:
|
|
1453
|
+
- XXX: window_size (3-digit zero-padded)
|
|
1454
|
+
- YY: num_layers (2-digit zero-padded)
|
|
1455
|
+
- ZZ: num_filters (2-digit zero-padded)
|
|
1456
|
+
- ZZ: kernel_size (2-digit zero-padded)
|
|
1457
|
+
- XXX: num_epochs (3-digit zero-padded)
|
|
1458
|
+
- Y: excludethresh (single digit)
|
|
1459
|
+
- Z: corrthresh (single digit)
|
|
1460
|
+
- Z: step (single digit)
|
|
1461
|
+
- Z: dilation_rate (single digit)
|
|
1462
|
+
|
|
1463
|
+
Examples
|
|
1464
|
+
--------
|
|
1465
|
+
>>> model = MyModel()
|
|
1466
|
+
>>> model.window_size = 100
|
|
1467
|
+
>>> model.num_layers = 5
|
|
1468
|
+
>>> model.num_filters = 32
|
|
1469
|
+
>>> model.kernel_size = 3
|
|
1470
|
+
>>> model.num_epochs = 1000
|
|
1471
|
+
>>> model.excludethresh = 0.5
|
|
1472
|
+
>>> model.corrthresh = 0.8
|
|
1473
|
+
>>> model.step = 1
|
|
1474
|
+
>>> model.dilation_rate = 2
|
|
1475
|
+
>>> model.activation = "relu"
|
|
1476
|
+
>>> model.modelroot = "./models"
|
|
1477
|
+
>>> model.getname()
|
|
1478
|
+
>>> print(model.modelname)
|
|
1479
|
+
'model_cnn_tf2_w100_l05_fn32_fl03_e1000_t05_ct08_s1_d2_relu'
|
|
1480
|
+
"""
|
|
561
1481
|
self.modelname = "_".join(
|
|
562
1482
|
[
|
|
563
1483
|
"model",
|
|
564
1484
|
"cnn",
|
|
1485
|
+
"tf2",
|
|
565
1486
|
"w" + str(self.window_size).zfill(3),
|
|
566
1487
|
"l" + str(self.num_layers).zfill(2),
|
|
567
1488
|
"fn" + str(self.num_filters).zfill(2),
|
|
@@ -588,6 +1509,63 @@ class CNNDLFilter(DeepLearningFilter):
|
|
|
588
1509
|
pass
|
|
589
1510
|
|
|
590
1511
|
def makenet(self):
|
|
1512
|
+
"""
|
|
1513
|
+
Create and configure a 1D convolutional neural network model.
|
|
1514
|
+
|
|
1515
|
+
This method builds a sequential CNN architecture with multiple convolutional layers,
|
|
1516
|
+
batch normalization, dropout regularization, and ReLU activation functions. The network
|
|
1517
|
+
is designed for sequence-to-sequence mapping with skip connections.
|
|
1518
|
+
|
|
1519
|
+
Parameters
|
|
1520
|
+
----------
|
|
1521
|
+
self : object
|
|
1522
|
+
The instance of the class containing the model configuration parameters.
|
|
1523
|
+
Expected attributes include:
|
|
1524
|
+
- num_filters : int
|
|
1525
|
+
Number of convolutional filters in each layer.
|
|
1526
|
+
- kernel_size : int
|
|
1527
|
+
Size of the convolutional kernel.
|
|
1528
|
+
- inputsize : int
|
|
1529
|
+
Size of the input sequence.
|
|
1530
|
+
- num_layers : int
|
|
1531
|
+
Total number of layers in the network.
|
|
1532
|
+
- dilation_rate : int
|
|
1533
|
+
Dilation rate for dilated convolutions.
|
|
1534
|
+
- dropout_rate : float
|
|
1535
|
+
Dropout rate for regularization.
|
|
1536
|
+
- activation : str or callable
|
|
1537
|
+
Activation function to use.
|
|
1538
|
+
|
|
1539
|
+
Returns
|
|
1540
|
+
-------
|
|
1541
|
+
None
|
|
1542
|
+
This method modifies the instance's model attribute in-place and does not return anything.
|
|
1543
|
+
|
|
1544
|
+
Notes
|
|
1545
|
+
-----
|
|
1546
|
+
The network architecture follows this pattern:
|
|
1547
|
+
- Input layer with Conv1D, BatchNormalization, Dropout, and Activation
|
|
1548
|
+
- Intermediate layers with Conv1D, BatchNormalization, Dropout, and Activation
|
|
1549
|
+
- Output layer with Conv1D matching the input size
|
|
1550
|
+
- Model compiled with RMSprop optimizer and MSE loss
|
|
1551
|
+
|
|
1552
|
+
Examples
|
|
1553
|
+
--------
|
|
1554
|
+
>>> class MyModel:
|
|
1555
|
+
... def __init__(self):
|
|
1556
|
+
... self.num_filters = 64
|
|
1557
|
+
... self.kernel_size = 3
|
|
1558
|
+
... self.inputsize = 100
|
|
1559
|
+
... self.num_layers = 5
|
|
1560
|
+
... self.dilation_rate = 2
|
|
1561
|
+
... self.dropout_rate = 0.2
|
|
1562
|
+
... self.activation = 'relu'
|
|
1563
|
+
... self.model = None
|
|
1564
|
+
...
|
|
1565
|
+
>>> model = MyModel()
|
|
1566
|
+
>>> model.makenet()
|
|
1567
|
+
>>> print(model.model.summary())
|
|
1568
|
+
"""
|
|
591
1569
|
self.model = Sequential()
|
|
592
1570
|
|
|
593
1571
|
# make the input layer
|
|
@@ -625,17 +1603,108 @@ class CNNDLFilter(DeepLearningFilter):
|
|
|
625
1603
|
|
|
626
1604
|
|
|
627
1605
|
class DenseAutoencoderDLFilter(DeepLearningFilter):
|
|
628
|
-
def __init__(self, encoding_dim=10, *args, **kwargs):
|
|
1606
|
+
def __init__(self, encoding_dim: int = 10, *args, **kwargs) -> None:
|
|
1607
|
+
"""
|
|
1608
|
+
Initialize the DenseAutoencoderDLFilter.
|
|
1609
|
+
|
|
1610
|
+
Parameters
|
|
1611
|
+
----------
|
|
1612
|
+
encoding_dim : int, default=10
|
|
1613
|
+
The dimensionality of the encoding layer in the autoencoder.
|
|
1614
|
+
*args
|
|
1615
|
+
Variable length argument list passed to the parent class constructor.
|
|
1616
|
+
**kwargs
|
|
1617
|
+
Arbitrary keyword arguments passed to the parent class constructor.
|
|
1618
|
+
|
|
1619
|
+
Returns
|
|
1620
|
+
-------
|
|
1621
|
+
None
|
|
1622
|
+
This method initializes the instance and does not return any value.
|
|
1623
|
+
|
|
1624
|
+
Notes
|
|
1625
|
+
-----
|
|
1626
|
+
This constructor sets up the autoencoder architecture by:
|
|
1627
|
+
1. Storing the encoding dimension as an instance attribute
|
|
1628
|
+
2. Updating the infodict with network type and encoding dimension information
|
|
1629
|
+
3. Calling the parent class constructor with passed arguments
|
|
1630
|
+
|
|
1631
|
+
Examples
|
|
1632
|
+
--------
|
|
1633
|
+
>>> filter = DenseAutoencoderDLFilter(encoding_dim=20)
|
|
1634
|
+
>>> print(filter.encoding_dim)
|
|
1635
|
+
20
|
|
1636
|
+
"""
|
|
629
1637
|
self.encoding_dim = encoding_dim
|
|
630
1638
|
self.infodict["nettype"] = "autoencoder"
|
|
631
1639
|
self.infodict["encoding_dim"] = self.encoding_dim
|
|
632
1640
|
super(DenseAutoencoderDLFilter, self).__init__(*args, **kwargs)
|
|
633
1641
|
|
|
634
1642
|
def getname(self):
|
|
1643
|
+
"""
|
|
1644
|
+
Generate and return model name based on configuration parameters.
|
|
1645
|
+
|
|
1646
|
+
This method constructs a descriptive model name by joining various configuration
|
|
1647
|
+
parameters with specific prefixes and formatting conventions. The generated name
|
|
1648
|
+
is used to create a unique identifier for the model and its corresponding directory
|
|
1649
|
+
path.
|
|
1650
|
+
|
|
1651
|
+
Parameters
|
|
1652
|
+
----------
|
|
1653
|
+
self : object
|
|
1654
|
+
The instance of the class containing the model configuration attributes.
|
|
1655
|
+
Expected attributes include:
|
|
1656
|
+
- `window_size`: int, size of the window
|
|
1657
|
+
- `encoding_dim`: int, dimension of the encoding layer
|
|
1658
|
+
- `num_epochs`: int, number of training epochs
|
|
1659
|
+
- `excludethresh`: float, threshold for exclusion
|
|
1660
|
+
- `corrthresh`: float, correlation threshold
|
|
1661
|
+
- `step`: int, step size
|
|
1662
|
+
- `activation`: str, activation function name
|
|
1663
|
+
- `modelroot`: str, root directory for models
|
|
1664
|
+
- `usebadpts`: bool, flag to include bad points
|
|
1665
|
+
- `excludebysubject`: bool, flag to exclude by subject
|
|
1666
|
+
- `namesuffix`: str, optional suffix to append to the model name
|
|
1667
|
+
|
|
1668
|
+
Returns
|
|
1669
|
+
-------
|
|
1670
|
+
None
|
|
1671
|
+
This method does not return a value but sets the following attributes:
|
|
1672
|
+
- `self.modelname`: str, the generated model name
|
|
1673
|
+
- `self.modelpath`: str, the full path to the model directory
|
|
1674
|
+
|
|
1675
|
+
Notes
|
|
1676
|
+
-----
|
|
1677
|
+
The model name is constructed using the following components:
|
|
1678
|
+
- "model_denseautoencoder_tf2" as base identifier
|
|
1679
|
+
- Window size with 3-digit zero-padded formatting (wXXX)
|
|
1680
|
+
- Encoding dimension with 3-digit zero-padded formatting (enXXX)
|
|
1681
|
+
- Number of epochs with 3-digit zero-padded formatting (eXXX)
|
|
1682
|
+
- Exclusion threshold (tX.XX)
|
|
1683
|
+
- Correlation threshold (ctX.XX)
|
|
1684
|
+
- Step size (sX)
|
|
1685
|
+
- Activation function name
|
|
1686
|
+
- Optional suffixes based on configuration flags
|
|
1687
|
+
|
|
1688
|
+
Examples
|
|
1689
|
+
--------
|
|
1690
|
+
>>> model = MyModel()
|
|
1691
|
+
>>> model.window_size = 100
|
|
1692
|
+
>>> model.encoding_dim = 50
|
|
1693
|
+
>>> model.num_epochs = 1000
|
|
1694
|
+
>>> model.excludethresh = 0.5
|
|
1695
|
+
>>> model.corrthresh = 0.8
|
|
1696
|
+
>>> model.step = 10
|
|
1697
|
+
>>> model.activation = 'relu'
|
|
1698
|
+
>>> model.modelroot = '/models'
|
|
1699
|
+
>>> model.getname()
|
|
1700
|
+
>>> print(model.modelname)
|
|
1701
|
+
'model_denseautoencoder_tf2_w100_en050_e1000_t0.5_ct00.8_s10_relu'
|
|
1702
|
+
"""
|
|
635
1703
|
self.modelname = "_".join(
|
|
636
1704
|
[
|
|
637
1705
|
"model",
|
|
638
1706
|
"denseautoencoder",
|
|
1707
|
+
"tf2",
|
|
639
1708
|
"w" + str(self.window_size).zfill(3),
|
|
640
1709
|
"en" + str(self.encoding_dim).zfill(3),
|
|
641
1710
|
"e" + str(self.num_epochs).zfill(3),
|
|
@@ -659,6 +1728,56 @@ class DenseAutoencoderDLFilter(DeepLearningFilter):
|
|
|
659
1728
|
pass
|
|
660
1729
|
|
|
661
1730
|
def makenet(self):
|
|
1731
|
+
"""
|
|
1732
|
+
Create and compile a neural network model for autoencoding.
|
|
1733
|
+
|
|
1734
|
+
This function constructs a symmetric autoencoder architecture with configurable
|
|
1735
|
+
number of layers, encoding dimension, and activation function. The network
|
|
1736
|
+
consists of an input layer, multiple encoding layers, an encoding layer,
|
|
1737
|
+
decoding layers, and an output layer. Batch normalization, dropout, and
|
|
1738
|
+
activation functions are applied at each layer as specified by the instance
|
|
1739
|
+
attributes.
|
|
1740
|
+
|
|
1741
|
+
Parameters
|
|
1742
|
+
----------
|
|
1743
|
+
self : object
|
|
1744
|
+
The instance of the class containing the following attributes:
|
|
1745
|
+
- num_layers : int
|
|
1746
|
+
Number of layers in the network.
|
|
1747
|
+
- encoding_dim : int
|
|
1748
|
+
Dimension of the encoding layer.
|
|
1749
|
+
- inputsize : int
|
|
1750
|
+
Size of the input data.
|
|
1751
|
+
- dropout_rate : float
|
|
1752
|
+
Dropout rate applied to all layers.
|
|
1753
|
+
- activation : str or callable
|
|
1754
|
+
Activation function used in all layers.
|
|
1755
|
+
- model : keras.Sequential
|
|
1756
|
+
The constructed Keras model (assigned within the function).
|
|
1757
|
+
|
|
1758
|
+
Returns
|
|
1759
|
+
-------
|
|
1760
|
+
None
|
|
1761
|
+
This function does not return a value but assigns the constructed model
|
|
1762
|
+
to the instance attribute `self.model`.
|
|
1763
|
+
|
|
1764
|
+
Notes
|
|
1765
|
+
-----
|
|
1766
|
+
- The network architecture is symmetric, with the number of units in each
|
|
1767
|
+
layer following a pattern that doubles or halves based on the layer index.
|
|
1768
|
+
- The model is compiled with the RMSprop optimizer and mean squared error loss.
|
|
1769
|
+
|
|
1770
|
+
Examples
|
|
1771
|
+
--------
|
|
1772
|
+
>>> autoencoder = AutoEncoder()
|
|
1773
|
+
>>> autoencoder.num_layers = 4
|
|
1774
|
+
>>> autoencoder.encoding_dim = 32
|
|
1775
|
+
>>> autoencoder.inputsize = 784
|
|
1776
|
+
>>> autoencoder.dropout_rate = 0.2
|
|
1777
|
+
>>> autoencoder.activation = 'relu'
|
|
1778
|
+
>>> autoencoder.makenet()
|
|
1779
|
+
>>> autoencoder.model.summary()
|
|
1780
|
+
"""
|
|
662
1781
|
self.model = Sequential()
|
|
663
1782
|
|
|
664
1783
|
# make the input layer
|
|
@@ -705,8 +1824,51 @@ class DenseAutoencoderDLFilter(DeepLearningFilter):
|
|
|
705
1824
|
|
|
706
1825
|
class ConvAutoencoderDLFilter(DeepLearningFilter):
|
|
707
1826
|
def __init__(
|
|
708
|
-
self,
|
|
709
|
-
|
|
1827
|
+
self,
|
|
1828
|
+
encoding_dim: int = 10,
|
|
1829
|
+
num_filters: int = 5,
|
|
1830
|
+
kernel_size: int = 5,
|
|
1831
|
+
dilation_rate: int = 1,
|
|
1832
|
+
*args,
|
|
1833
|
+
**kwargs,
|
|
1834
|
+
) -> None:
|
|
1835
|
+
"""
|
|
1836
|
+
Initialize ConvAutoencoderDLFilter.
|
|
1837
|
+
|
|
1838
|
+
Parameters
|
|
1839
|
+
----------
|
|
1840
|
+
encoding_dim : int, optional
|
|
1841
|
+
Dimension of the encoding layer, by default 10.
|
|
1842
|
+
num_filters : int, optional
|
|
1843
|
+
Number of filters in the convolutional layers, by default 5.
|
|
1844
|
+
kernel_size : int, optional
|
|
1845
|
+
Size of the convolutional kernel, by default 5.
|
|
1846
|
+
dilation_rate : int, optional
|
|
1847
|
+
Dilation rate for the convolutional layers, by default 1.
|
|
1848
|
+
*args
|
|
1849
|
+
Variable length argument list.
|
|
1850
|
+
**kwargs
|
|
1851
|
+
Arbitrary keyword arguments.
|
|
1852
|
+
|
|
1853
|
+
Returns
|
|
1854
|
+
-------
|
|
1855
|
+
None
|
|
1856
|
+
This method does not return any value.
|
|
1857
|
+
|
|
1858
|
+
Notes
|
|
1859
|
+
-----
|
|
1860
|
+
This constructor initializes the ConvAutoencoderDLFilter with specified
|
|
1861
|
+
convolutional parameters and sets up the infodict with network metadata.
|
|
1862
|
+
|
|
1863
|
+
Examples
|
|
1864
|
+
--------
|
|
1865
|
+
>>> model = ConvAutoencoderDLFilter(
|
|
1866
|
+
... encoding_dim=15,
|
|
1867
|
+
... num_filters=8,
|
|
1868
|
+
... kernel_size=3,
|
|
1869
|
+
... dilation_rate=2
|
|
1870
|
+
... )
|
|
1871
|
+
"""
|
|
710
1872
|
self.encoding_dim = encoding_dim
|
|
711
1873
|
self.num_filters = num_filters
|
|
712
1874
|
self.kernel_size = kernel_size
|
|
@@ -718,10 +1880,72 @@ class ConvAutoencoderDLFilter(DeepLearningFilter):
|
|
|
718
1880
|
super(ConvAutoencoderDLFilter, self).__init__(*args, **kwargs)
|
|
719
1881
|
|
|
720
1882
|
def getname(self):
|
|
1883
|
+
"""
|
|
1884
|
+
Generate and configure the model name and path based on current configuration parameters.
|
|
1885
|
+
|
|
1886
|
+
This method constructs a descriptive model name string using various configuration
|
|
1887
|
+
parameters and creates the corresponding directory path for model storage. The generated
|
|
1888
|
+
name includes information about window size, encoding dimensions, filter settings,
|
|
1889
|
+
training parameters, and optional flags.
|
|
1890
|
+
|
|
1891
|
+
Parameters
|
|
1892
|
+
----------
|
|
1893
|
+
self : object
|
|
1894
|
+
The instance containing configuration parameters for model naming.
|
|
1895
|
+
Expected attributes include:
|
|
1896
|
+
- window_size : int
|
|
1897
|
+
- encoding_dim : int
|
|
1898
|
+
- num_filters : int
|
|
1899
|
+
- kernel_size : int
|
|
1900
|
+
- num_epochs : int
|
|
1901
|
+
- excludethresh : float
|
|
1902
|
+
- corrthresh : float
|
|
1903
|
+
- step : int
|
|
1904
|
+
- activation : str
|
|
1905
|
+
- usebadpts : bool
|
|
1906
|
+
- excludebysubject : bool
|
|
1907
|
+
- namesuffix : str or None
|
|
1908
|
+
- modelroot : str
|
|
1909
|
+
|
|
1910
|
+
Returns
|
|
1911
|
+
-------
|
|
1912
|
+
None
|
|
1913
|
+
This method modifies the instance attributes in-place and does not return a value.
|
|
1914
|
+
Modifies the following instance attributes:
|
|
1915
|
+
- modelname : str
|
|
1916
|
+
- modelpath : str
|
|
1917
|
+
|
|
1918
|
+
Notes
|
|
1919
|
+
-----
|
|
1920
|
+
The generated model name follows a specific naming convention:
|
|
1921
|
+
"model_convautoencoder_tf2_wXXX_enYYY_fnZZ_flZZ_eXXX_tY_ctZ_sZ_activation"
|
|
1922
|
+
where XXX, YYY, ZZ, and Z represent zero-padded numerical values.
|
|
1923
|
+
|
|
1924
|
+
Examples
|
|
1925
|
+
--------
|
|
1926
|
+
>>> model = MyModel()
|
|
1927
|
+
>>> model.window_size = 100
|
|
1928
|
+
>>> model.encoding_dim = 50
|
|
1929
|
+
>>> model.num_filters = 32
|
|
1930
|
+
>>> model.kernel_size = 5
|
|
1931
|
+
>>> model.num_epochs = 1000
|
|
1932
|
+
>>> model.excludethresh = 0.5
|
|
1933
|
+
>>> model.corrthresh = 0.8
|
|
1934
|
+
>>> model.step = 10
|
|
1935
|
+
>>> model.activation = 'relu'
|
|
1936
|
+
>>> model.usebadpts = True
|
|
1937
|
+
>>> model.excludebysubject = False
|
|
1938
|
+
>>> model.namesuffix = 'test'
|
|
1939
|
+
>>> model.modelroot = '/path/to/models'
|
|
1940
|
+
>>> model.getname()
|
|
1941
|
+
>>> print(model.modelname)
|
|
1942
|
+
'model_convautoencoder_tf2_w100_en050_fn32_fl05_e1000_t0.5_ct0.8_s10_relu_usebadpts_test'
|
|
1943
|
+
"""
|
|
721
1944
|
self.modelname = "_".join(
|
|
722
1945
|
[
|
|
723
1946
|
"model",
|
|
724
1947
|
"convautoencoder",
|
|
1948
|
+
"tf2",
|
|
725
1949
|
"w" + str(self.window_size).zfill(3),
|
|
726
1950
|
"en" + str(self.encoding_dim).zfill(3),
|
|
727
1951
|
"fn" + str(self.num_filters).zfill(2),
|
|
@@ -747,6 +1971,60 @@ class ConvAutoencoderDLFilter(DeepLearningFilter):
|
|
|
747
1971
|
pass
|
|
748
1972
|
|
|
749
1973
|
def makenet(self):
|
|
1974
|
+
"""
|
|
1975
|
+
Build and compile a 1D convolutional autoencoder model.
|
|
1976
|
+
|
|
1977
|
+
This function constructs a symmetric encoder-decoder architecture using 1D convolutions,
|
|
1978
|
+
batch normalization, dropout, and pooling layers. The model is designed for sequence
|
|
1979
|
+
processing tasks such as time-series or signal denoising.
|
|
1980
|
+
|
|
1981
|
+
Parameters
|
|
1982
|
+
----------
|
|
1983
|
+
self : object
|
|
1984
|
+
Instance of the class containing the following attributes:
|
|
1985
|
+
- window_size : int
|
|
1986
|
+
The length of the input sequence.
|
|
1987
|
+
- inputsize : int
|
|
1988
|
+
The number of input features at each time step.
|
|
1989
|
+
- num_filters : int
|
|
1990
|
+
Initial number of filters in the first convolutional layer.
|
|
1991
|
+
- kernel_size : int
|
|
1992
|
+
Size of the convolutional kernel.
|
|
1993
|
+
- dropout_rate : float
|
|
1994
|
+
Dropout rate applied after batch normalization.
|
|
1995
|
+
- activation : str or callable
|
|
1996
|
+
Activation function used in layers.
|
|
1997
|
+
- encoding_dim : int
|
|
1998
|
+
Dimensionality of the encoded representation.
|
|
1999
|
+
|
|
2000
|
+
Returns
|
|
2001
|
+
-------
|
|
2002
|
+
None
|
|
2003
|
+
This method does not return a value but sets the `self.model` attribute to the
|
|
2004
|
+
compiled Keras model.
|
|
2005
|
+
|
|
2006
|
+
Notes
|
|
2007
|
+
-----
|
|
2008
|
+
The model architecture includes:
|
|
2009
|
+
- An initial convolutional block followed by max pooling.
|
|
2010
|
+
- Three encoding layers with increasing filter sizes and halving the sequence length.
|
|
2011
|
+
- A bottleneck layer that compresses the representation.
|
|
2012
|
+
- A symmetric decoding path that mirrors the encoding path.
|
|
2013
|
+
- Final upsampling to reconstruct the original sequence dimensions.
|
|
2014
|
+
|
|
2015
|
+
Examples
|
|
2016
|
+
--------
|
|
2017
|
+
>>> model = MyClass()
|
|
2018
|
+
>>> model.window_size = 100
|
|
2019
|
+
>>> model.inputsize = 10
|
|
2020
|
+
>>> model.num_filters = 32
|
|
2021
|
+
>>> model.kernel_size = 3
|
|
2022
|
+
>>> model.dropout_rate = 0.2
|
|
2023
|
+
>>> model.activation = 'relu'
|
|
2024
|
+
>>> model.encoding_dim = 16
|
|
2025
|
+
>>> model.makenet()
|
|
2026
|
+
>>> model.model.summary()
|
|
2027
|
+
"""
|
|
750
2028
|
input_layer = Input(shape=(self.window_size, self.inputsize))
|
|
751
2029
|
x = input_layer
|
|
752
2030
|
|
|
@@ -803,8 +2081,51 @@ class ConvAutoencoderDLFilter(DeepLearningFilter):
|
|
|
803
2081
|
|
|
804
2082
|
class CRNNDLFilter(DeepLearningFilter):
|
|
805
2083
|
def __init__(
|
|
806
|
-
self,
|
|
807
|
-
|
|
2084
|
+
self,
|
|
2085
|
+
encoding_dim: int = 10,
|
|
2086
|
+
num_filters: int = 10,
|
|
2087
|
+
kernel_size: int = 5,
|
|
2088
|
+
dilation_rate: int = 1,
|
|
2089
|
+
*args,
|
|
2090
|
+
**kwargs,
|
|
2091
|
+
) -> None:
|
|
2092
|
+
"""
|
|
2093
|
+
Initialize CRNNDLFilter layer.
|
|
2094
|
+
|
|
2095
|
+
Parameters
|
|
2096
|
+
----------
|
|
2097
|
+
encoding_dim : int, default=10
|
|
2098
|
+
Dimension of the encoding layer.
|
|
2099
|
+
num_filters : int, default=10
|
|
2100
|
+
Number of filters in the convolutional layer.
|
|
2101
|
+
kernel_size : int, default=5
|
|
2102
|
+
Size of the convolutional kernel.
|
|
2103
|
+
dilation_rate : int, default=1
|
|
2104
|
+
Dilation rate for the convolutional layer.
|
|
2105
|
+
*args
|
|
2106
|
+
Variable length argument list.
|
|
2107
|
+
**kwargs
|
|
2108
|
+
Arbitrary keyword arguments.
|
|
2109
|
+
|
|
2110
|
+
Returns
|
|
2111
|
+
-------
|
|
2112
|
+
None
|
|
2113
|
+
This method does not return any value.
|
|
2114
|
+
|
|
2115
|
+
Notes
|
|
2116
|
+
-----
|
|
2117
|
+
This constructor initializes the CRNNDLFilter layer with specified parameters
|
|
2118
|
+
and sets up the network type information in the infodict.
|
|
2119
|
+
|
|
2120
|
+
Examples
|
|
2121
|
+
--------
|
|
2122
|
+
>>> filter_layer = CRNNDLFilter(
|
|
2123
|
+
... encoding_dim=15,
|
|
2124
|
+
... num_filters=20,
|
|
2125
|
+
... kernel_size=3,
|
|
2126
|
+
... dilation_rate=2
|
|
2127
|
+
... )
|
|
2128
|
+
"""
|
|
808
2129
|
self.num_filters = num_filters
|
|
809
2130
|
self.kernel_size = kernel_size
|
|
810
2131
|
self.dilation_rate = dilation_rate
|
|
@@ -816,10 +2137,73 @@ class CRNNDLFilter(DeepLearningFilter):
|
|
|
816
2137
|
super(CRNNDLFilter, self).__init__(*args, **kwargs)
|
|
817
2138
|
|
|
818
2139
|
def getname(self):
|
|
2140
|
+
"""
|
|
2141
|
+
Generate and configure the model name and path based on current parameters.
|
|
2142
|
+
|
|
2143
|
+
This method constructs a descriptive model name string using various instance
|
|
2144
|
+
attributes and creates the corresponding directory path for model storage.
|
|
2145
|
+
|
|
2146
|
+
Parameters
|
|
2147
|
+
----------
|
|
2148
|
+
self : object
|
|
2149
|
+
The instance containing model configuration parameters
|
|
2150
|
+
|
|
2151
|
+
Attributes Used
|
|
2152
|
+
---------------
|
|
2153
|
+
window_size : int
|
|
2154
|
+
Size of the sliding window
|
|
2155
|
+
encoding_dim : int
|
|
2156
|
+
Dimension of the encoding layer
|
|
2157
|
+
num_filters : int
|
|
2158
|
+
Number of filters in the convolutional layers
|
|
2159
|
+
kernel_size : int
|
|
2160
|
+
Size of the convolutional kernel
|
|
2161
|
+
num_epochs : int
|
|
2162
|
+
Number of training epochs
|
|
2163
|
+
excludethresh : float
|
|
2164
|
+
Threshold for excluding data points
|
|
2165
|
+
corrthresh : float
|
|
2166
|
+
Correlation threshold for filtering
|
|
2167
|
+
step : int
|
|
2168
|
+
Step size for sliding window
|
|
2169
|
+
activation : str
|
|
2170
|
+
Activation function used
|
|
2171
|
+
usebadpts : bool
|
|
2172
|
+
Whether to use bad points in training
|
|
2173
|
+
excludebysubject : bool
|
|
2174
|
+
Whether to exclude data by subject
|
|
2175
|
+
namesuffix : str, optional
|
|
2176
|
+
Additional suffix to append to model name
|
|
2177
|
+
modelroot : str
|
|
2178
|
+
Root directory for model storage
|
|
2179
|
+
|
|
2180
|
+
Returns
|
|
2181
|
+
-------
|
|
2182
|
+
None
|
|
2183
|
+
This method modifies instance attributes in-place:
|
|
2184
|
+
- self.modelname: constructed model name string
|
|
2185
|
+
- self.modelpath: full path to model directory
|
|
2186
|
+
|
|
2187
|
+
Notes
|
|
2188
|
+
-----
|
|
2189
|
+
The generated model name follows a consistent format:
|
|
2190
|
+
"model_crnn_tf2_wXxx_enXxx_fnXX_flXX_eXXX_tX_ctX_sX_activation"
|
|
2191
|
+
where Xxx represents zero-padded numbers and XX represents zero-padded two-digit numbers.
|
|
2192
|
+
|
|
2193
|
+
Examples
|
|
2194
|
+
--------
|
|
2195
|
+
>>> model = MyModel()
|
|
2196
|
+
>>> model.window_size = 100
|
|
2197
|
+
>>> model.encoding_dim = 50
|
|
2198
|
+
>>> model.getname()
|
|
2199
|
+
>>> print(model.modelname)
|
|
2200
|
+
'model_crnn_tf2_w100_en050_fn10_fl10_e001_t0.5_ct0.8_s1_relu'
|
|
2201
|
+
"""
|
|
819
2202
|
self.modelname = "_".join(
|
|
820
2203
|
[
|
|
821
2204
|
"model",
|
|
822
2205
|
"crnn",
|
|
2206
|
+
"tf2",
|
|
823
2207
|
"w" + str(self.window_size).zfill(3),
|
|
824
2208
|
"en" + str(self.encoding_dim).zfill(3),
|
|
825
2209
|
"fn" + str(self.num_filters).zfill(2),
|
|
@@ -845,6 +2229,65 @@ class CRNNDLFilter(DeepLearningFilter):
|
|
|
845
2229
|
pass
|
|
846
2230
|
|
|
847
2231
|
def makenet(self):
|
|
2232
|
+
"""
|
|
2233
|
+
Create and compile a 1D convolutional neural network for temporal feature extraction and reconstruction.
|
|
2234
|
+
|
|
2235
|
+
This function builds a neural network architecture consisting of:
|
|
2236
|
+
- A convolutional front-end for feature extraction
|
|
2237
|
+
- A bidirectional LSTM layer for temporal modeling
|
|
2238
|
+
- A dense output layer mapping to the input size
|
|
2239
|
+
|
|
2240
|
+
The model is compiled with Adam optimizer and mean squared error loss.
|
|
2241
|
+
|
|
2242
|
+
Parameters
|
|
2243
|
+
----------
|
|
2244
|
+
self : object
|
|
2245
|
+
The class instance containing the following attributes:
|
|
2246
|
+
- window_size : int
|
|
2247
|
+
The size of the input time window.
|
|
2248
|
+
- inputsize : int
|
|
2249
|
+
The number of input channels/features.
|
|
2250
|
+
- num_filters : int
|
|
2251
|
+
Number of filters in the convolutional layers.
|
|
2252
|
+
- kernel_size : int
|
|
2253
|
+
Size of the convolutional kernel.
|
|
2254
|
+
- dropout_rate : float
|
|
2255
|
+
Dropout rate for regularization.
|
|
2256
|
+
- activation : str or callable
|
|
2257
|
+
Activation function for convolutional layers.
|
|
2258
|
+
- encoding_dim : int
|
|
2259
|
+
Number of units in the LSTM layer.
|
|
2260
|
+
|
|
2261
|
+
Returns
|
|
2262
|
+
-------
|
|
2263
|
+
None
|
|
2264
|
+
This method modifies the instance's `model` attribute in-place.
|
|
2265
|
+
|
|
2266
|
+
Notes
|
|
2267
|
+
-----
|
|
2268
|
+
The network architecture follows this pipeline:
|
|
2269
|
+
Input -> Conv1D -> BatchNorm -> Dropout -> Activation ->
|
|
2270
|
+
Conv1D -> BatchNorm -> Dropout -> Activation ->
|
|
2271
|
+
Bidirectional LSTM -> Dense -> Output
|
|
2272
|
+
|
|
2273
|
+
The model is compiled with:
|
|
2274
|
+
- Optimizer: Adam
|
|
2275
|
+
- Loss: Mean Squared Error (MSE)
|
|
2276
|
+
|
|
2277
|
+
Examples
|
|
2278
|
+
--------
|
|
2279
|
+
>>> # Assuming a class with the required attributes
|
|
2280
|
+
>>> model = MyModel()
|
|
2281
|
+
>>> model.window_size = 100
|
|
2282
|
+
>>> model.inputsize = 10
|
|
2283
|
+
>>> model.num_filters = 32
|
|
2284
|
+
>>> model.kernel_size = 3
|
|
2285
|
+
>>> model.dropout_rate = 0.2
|
|
2286
|
+
>>> model.activation = 'relu'
|
|
2287
|
+
>>> model.encoding_dim = 64
|
|
2288
|
+
>>> model.makenet()
|
|
2289
|
+
>>> model.model.summary()
|
|
2290
|
+
"""
|
|
848
2291
|
input_layer = Input(shape=(self.window_size, self.inputsize))
|
|
849
2292
|
x = input_layer
|
|
850
2293
|
|
|
@@ -875,17 +2318,110 @@ class CRNNDLFilter(DeepLearningFilter):
|
|
|
875
2318
|
|
|
876
2319
|
|
|
877
2320
|
class LSTMDLFilter(DeepLearningFilter):
|
|
878
|
-
def __init__(self, num_units=16, *args, **kwargs):
|
|
2321
|
+
def __init__(self, num_units: int = 16, *args, **kwargs) -> None:
|
|
2322
|
+
"""
|
|
2323
|
+
Initialize the LSTMDLFilter layer.
|
|
2324
|
+
|
|
2325
|
+
Parameters
|
|
2326
|
+
----------
|
|
2327
|
+
num_units : int, optional
|
|
2328
|
+
Number of units in the LSTM layer, by default 16
|
|
2329
|
+
*args : tuple
|
|
2330
|
+
Additional positional arguments passed to the parent class
|
|
2331
|
+
**kwargs : dict
|
|
2332
|
+
Additional keyword arguments passed to the parent class
|
|
2333
|
+
|
|
2334
|
+
Returns
|
|
2335
|
+
-------
|
|
2336
|
+
None
|
|
2337
|
+
This method initializes the layer in-place and does not return any value
|
|
2338
|
+
|
|
2339
|
+
Notes
|
|
2340
|
+
-----
|
|
2341
|
+
This method sets up the LSTM layer with the specified number of units and
|
|
2342
|
+
configures the infodict with the network type and unit count. The parent
|
|
2343
|
+
class initialization is called with any additional arguments provided.
|
|
2344
|
+
|
|
2345
|
+
Examples
|
|
2346
|
+
--------
|
|
2347
|
+
>>> layer = LSTMDLFilter(num_units=32)
|
|
2348
|
+
>>> print(layer.num_units)
|
|
2349
|
+
32
|
|
2350
|
+
"""
|
|
879
2351
|
self.num_units = num_units
|
|
880
2352
|
self.infodict["nettype"] = "lstm"
|
|
881
2353
|
self.infodict["num_units"] = self.num_units
|
|
882
2354
|
super(LSTMDLFilter, self).__init__(*args, **kwargs)
|
|
883
2355
|
|
|
884
2356
|
def getname(self):
|
|
2357
|
+
"""
|
|
2358
|
+
Generate and return the model name and path based on current configuration parameters.
|
|
2359
|
+
|
|
2360
|
+
This method constructs a standardized model name string using various configuration
|
|
2361
|
+
parameters and creates the corresponding directory path. The generated name includes
|
|
2362
|
+
information about the model architecture, training parameters, and preprocessing settings.
|
|
2363
|
+
|
|
2364
|
+
Parameters
|
|
2365
|
+
----------
|
|
2366
|
+
self : object
|
|
2367
|
+
The instance containing the following attributes:
|
|
2368
|
+
- window_size : int
|
|
2369
|
+
Size of the sliding window for time series data
|
|
2370
|
+
- num_layers : int
|
|
2371
|
+
Number of LSTM layers in the model
|
|
2372
|
+
- num_units : int
|
|
2373
|
+
Number of units in each LSTM layer
|
|
2374
|
+
- dropout_rate : float
|
|
2375
|
+
Dropout rate for regularization
|
|
2376
|
+
- num_epochs : int
|
|
2377
|
+
Number of training epochs
|
|
2378
|
+
- excludethresh : float
|
|
2379
|
+
Threshold for exclusion criteria
|
|
2380
|
+
- corrthresh : float
|
|
2381
|
+
Correlation threshold for filtering
|
|
2382
|
+
- step : int
|
|
2383
|
+
Step size for data processing
|
|
2384
|
+
- excludebysubject : bool
|
|
2385
|
+
Whether to exclude data by subject
|
|
2386
|
+
- modelroot : str
|
|
2387
|
+
Root directory for model storage
|
|
2388
|
+
|
|
2389
|
+
Returns
|
|
2390
|
+
-------
|
|
2391
|
+
None
|
|
2392
|
+
This method modifies the instance attributes `modelname` and `modelpath` in place.
|
|
2393
|
+
It does not return any value.
|
|
2394
|
+
|
|
2395
|
+
Notes
|
|
2396
|
+
-----
|
|
2397
|
+
The generated model name follows a specific naming convention:
|
|
2398
|
+
"model_lstm_tf2_wXxx_lYY_nuZZZ_dDD_rdDD_eEEE_tT_ctTT_sS"
|
|
2399
|
+
where Xxx, YY, ZZZ, DD, EEEE, T, TT, S represent zero-padded numerical values.
|
|
2400
|
+
|
|
2401
|
+
If `excludebysubject` is True, "_excludebysubject" is appended to the model name.
|
|
2402
|
+
|
|
2403
|
+
Examples
|
|
2404
|
+
--------
|
|
2405
|
+
>>> model = MyModel()
|
|
2406
|
+
>>> model.window_size = 100
|
|
2407
|
+
>>> model.num_layers = 2
|
|
2408
|
+
>>> model.num_units = 128
|
|
2409
|
+
>>> model.dropout_rate = 0.2
|
|
2410
|
+
>>> model.num_epochs = 100
|
|
2411
|
+
>>> model.excludethresh = 0.5
|
|
2412
|
+
>>> model.corrthresh = 0.8
|
|
2413
|
+
>>> model.step = 1
|
|
2414
|
+
>>> model.excludebysubject = True
|
|
2415
|
+
>>> model.modelroot = "/path/to/models"
|
|
2416
|
+
>>> model.getname()
|
|
2417
|
+
>>> print(model.modelname)
|
|
2418
|
+
'model_lstm_tf2_w100_l02_nu128_d02_rd02_e100_t05_ct08_s1_excludebysubject'
|
|
2419
|
+
"""
|
|
885
2420
|
self.modelname = "_".join(
|
|
886
2421
|
[
|
|
887
2422
|
"model",
|
|
888
2423
|
"lstm",
|
|
2424
|
+
"tf2",
|
|
889
2425
|
"w" + str(self.window_size).zfill(3),
|
|
890
2426
|
"l" + str(self.num_layers).zfill(2),
|
|
891
2427
|
"nu" + str(self.num_units),
|
|
@@ -907,6 +2443,47 @@ class LSTMDLFilter(DeepLearningFilter):
|
|
|
907
2443
|
pass
|
|
908
2444
|
|
|
909
2445
|
def makenet(self):
|
|
2446
|
+
"""
|
|
2447
|
+
Create and configure a bidirectional LSTM neural network model.
|
|
2448
|
+
|
|
2449
|
+
This function builds a sequential neural network architecture using bidirectional LSTM layers
|
|
2450
|
+
followed by time-distributed dense layers. The model is compiled with Adam optimizer and MSE loss.
|
|
2451
|
+
|
|
2452
|
+
Parameters
|
|
2453
|
+
----------
|
|
2454
|
+
self : object
|
|
2455
|
+
The instance containing the following attributes:
|
|
2456
|
+
- num_layers : int
|
|
2457
|
+
Number of LSTM layers in the model
|
|
2458
|
+
- num_units : int
|
|
2459
|
+
Number of units in each LSTM layer
|
|
2460
|
+
- dropout_rate : float
|
|
2461
|
+
Dropout rate for both dropout and recurrent dropout
|
|
2462
|
+
- window_size : int
|
|
2463
|
+
Size of the input window for time series data
|
|
2464
|
+
|
|
2465
|
+
Returns
|
|
2466
|
+
-------
|
|
2467
|
+
None
|
|
2468
|
+
This method modifies the instance in-place by setting the `model` attribute.
|
|
2469
|
+
|
|
2470
|
+
Notes
|
|
2471
|
+
-----
|
|
2472
|
+
The model architecture consists of:
|
|
2473
|
+
1. Bidirectional LSTM layers with specified number of units and dropout rates
|
|
2474
|
+
2. TimeDistributed Dense layers to map outputs back to window size
|
|
2475
|
+
3. Compilation with Adam optimizer and MSE loss function
|
|
2476
|
+
|
|
2477
|
+
Examples
|
|
2478
|
+
--------
|
|
2479
|
+
>>> model_instance = MyModel()
|
|
2480
|
+
>>> model_instance.num_layers = 2
|
|
2481
|
+
>>> model_instance.num_units = 50
|
|
2482
|
+
>>> model_instance.dropout_rate = 0.2
|
|
2483
|
+
>>> model_instance.window_size = 10
|
|
2484
|
+
>>> model_instance.makenet()
|
|
2485
|
+
>>> print(model_instance.model.summary())
|
|
2486
|
+
"""
|
|
910
2487
|
self.model = Sequential()
|
|
911
2488
|
|
|
912
2489
|
# each layer consists of an LSTM followed by a dense time distributed layer to get it back to the window size
|
|
@@ -928,7 +2505,53 @@ class LSTMDLFilter(DeepLearningFilter):
|
|
|
928
2505
|
|
|
929
2506
|
|
|
930
2507
|
class HybridDLFilter(DeepLearningFilter):
|
|
931
|
-
def __init__(
|
|
2508
|
+
def __init__(
|
|
2509
|
+
self,
|
|
2510
|
+
invert: bool = False,
|
|
2511
|
+
num_filters: int = 10,
|
|
2512
|
+
kernel_size: int = 5,
|
|
2513
|
+
num_units: int = 16,
|
|
2514
|
+
*args,
|
|
2515
|
+
**kwargs,
|
|
2516
|
+
) -> None:
|
|
2517
|
+
"""
|
|
2518
|
+
Initialize the HybridDLFilter layer.
|
|
2519
|
+
|
|
2520
|
+
Parameters
|
|
2521
|
+
----------
|
|
2522
|
+
invert : bool, optional
|
|
2523
|
+
If True, inverts the filter operation, by default False
|
|
2524
|
+
num_filters : int, optional
|
|
2525
|
+
Number of filters to use in the convolutional layer, by default 10
|
|
2526
|
+
kernel_size : int, optional
|
|
2527
|
+
Size of the convolutional kernel, by default 5
|
|
2528
|
+
num_units : int, optional
|
|
2529
|
+
Number of units in the dense layer, by default 16
|
|
2530
|
+
*args
|
|
2531
|
+
Variable length argument list
|
|
2532
|
+
**kwargs
|
|
2533
|
+
Arbitrary keyword arguments
|
|
2534
|
+
|
|
2535
|
+
Returns
|
|
2536
|
+
-------
|
|
2537
|
+
None
|
|
2538
|
+
This method does not return a value
|
|
2539
|
+
|
|
2540
|
+
Notes
|
|
2541
|
+
-----
|
|
2542
|
+
This method initializes the hybrid deep learning filter by setting up
|
|
2543
|
+
the convolutional and dense layer parameters. The infodict is populated
|
|
2544
|
+
with configuration information for tracking and debugging purposes.
|
|
2545
|
+
|
|
2546
|
+
Examples
|
|
2547
|
+
--------
|
|
2548
|
+
>>> layer = HybridDLFilter(
|
|
2549
|
+
... invert=True,
|
|
2550
|
+
... num_filters=20,
|
|
2551
|
+
... kernel_size=3,
|
|
2552
|
+
... num_units=32
|
|
2553
|
+
... )
|
|
2554
|
+
"""
|
|
932
2555
|
self.invert = invert
|
|
933
2556
|
self.num_filters = num_filters
|
|
934
2557
|
self.kernel_size = kernel_size
|
|
@@ -941,10 +2564,84 @@ class HybridDLFilter(DeepLearningFilter):
|
|
|
941
2564
|
super(HybridDLFilter, self).__init__(*args, **kwargs)
|
|
942
2565
|
|
|
943
2566
|
def getname(self):
|
|
2567
|
+
"""
|
|
2568
|
+
Generate and return the model name and path based on current configuration parameters.
|
|
2569
|
+
|
|
2570
|
+
This method constructs a descriptive model name string by joining various configuration
|
|
2571
|
+
parameters with specific prefixes and formatting conventions. The resulting model name
|
|
2572
|
+
is used to create a unique directory path for model storage.
|
|
2573
|
+
|
|
2574
|
+
Parameters
|
|
2575
|
+
----------
|
|
2576
|
+
self : object
|
|
2577
|
+
The instance containing model configuration parameters. Expected attributes include:
|
|
2578
|
+
- `window_size` : int
|
|
2579
|
+
- `num_layers` : int
|
|
2580
|
+
- `num_filters` : int
|
|
2581
|
+
- `kernel_size` : int
|
|
2582
|
+
- `num_units` : int
|
|
2583
|
+
- `dropout_rate` : float
|
|
2584
|
+
- `num_epochs` : int
|
|
2585
|
+
- `excludethresh` : float
|
|
2586
|
+
- `corrthresh` : float
|
|
2587
|
+
- `step` : int
|
|
2588
|
+
- `activation` : str
|
|
2589
|
+
- `invert` : bool
|
|
2590
|
+
- `excludebysubject` : bool
|
|
2591
|
+
- `modelroot` : str
|
|
2592
|
+
|
|
2593
|
+
Returns
|
|
2594
|
+
-------
|
|
2595
|
+
None
|
|
2596
|
+
This method does not return a value but modifies instance attributes:
|
|
2597
|
+
- `self.modelname`: The constructed model name string
|
|
2598
|
+
- `self.modelpath`: The full path to the model directory
|
|
2599
|
+
|
|
2600
|
+
Notes
|
|
2601
|
+
-----
|
|
2602
|
+
The model name is constructed using the following components:
|
|
2603
|
+
- "model_hybrid_tf2_" prefix
|
|
2604
|
+
- Window size with 3-digit zero-padded formatting
|
|
2605
|
+
- Number of layers with 2-digit zero-padded formatting
|
|
2606
|
+
- Number of filters with 2-digit zero-padded formatting
|
|
2607
|
+
- Kernel size with 2-digit zero-padded formatting
|
|
2608
|
+
- Number of units
|
|
2609
|
+
- Dropout rate (appears twice with different prefixes)
|
|
2610
|
+
- Number of epochs with 3-digit zero-padded formatting
|
|
2611
|
+
- Exclusion threshold
|
|
2612
|
+
- Correlation threshold
|
|
2613
|
+
- Step size
|
|
2614
|
+
- Activation function name
|
|
2615
|
+
|
|
2616
|
+
Additional suffixes are appended based on boolean flags:
|
|
2617
|
+
- "_invert" if `self.invert` is True
|
|
2618
|
+
- "_excludebysubject" if `self.excludebysubject` is True
|
|
2619
|
+
|
|
2620
|
+
Examples
|
|
2621
|
+
--------
|
|
2622
|
+
>>> model = MyModel()
|
|
2623
|
+
>>> model.window_size = 100
|
|
2624
|
+
>>> model.num_layers = 3
|
|
2625
|
+
>>> model.num_filters = 16
|
|
2626
|
+
>>> model.kernel_size = 5
|
|
2627
|
+
>>> model.num_units = 64
|
|
2628
|
+
>>> model.dropout_rate = 0.2
|
|
2629
|
+
>>> model.num_epochs = 100
|
|
2630
|
+
>>> model.excludethresh = 0.5
|
|
2631
|
+
>>> model.corrthresh = 0.8
|
|
2632
|
+
>>> model.step = 1
|
|
2633
|
+
>>> model.activation = "relu"
|
|
2634
|
+
>>> model.invert = True
|
|
2635
|
+
>>> model.excludebysubject = False
|
|
2636
|
+
>>> model.getname()
|
|
2637
|
+
>>> print(model.modelname)
|
|
2638
|
+
'model_hybrid_tf2_w100_l03_fn16_fl05_nu64_d0.2_rd0.2_e100_t0.5_ct0.8_s01_relu_invert'
|
|
2639
|
+
"""
|
|
944
2640
|
self.modelname = "_".join(
|
|
945
2641
|
[
|
|
946
2642
|
"model",
|
|
947
2643
|
"hybrid",
|
|
2644
|
+
"tf2",
|
|
948
2645
|
"w" + str(self.window_size).zfill(3),
|
|
949
2646
|
"l" + str(self.num_layers).zfill(2),
|
|
950
2647
|
"fn" + str(self.num_filters).zfill(2),
|
|
@@ -971,6 +2668,71 @@ class HybridDLFilter(DeepLearningFilter):
|
|
|
971
2668
|
pass
|
|
972
2669
|
|
|
973
2670
|
def makenet(self):
|
|
2671
|
+
"""
|
|
2672
|
+
Build and compile a neural network model with configurable CNN and LSTM layers.
|
|
2673
|
+
|
|
2674
|
+
This function constructs a neural network model using Keras, with the architecture
|
|
2675
|
+
determined by the `invert` flag. If `invert` is True, the model starts with a
|
|
2676
|
+
Conv1D layer followed by LSTM layers; otherwise, it starts with LSTM layers
|
|
2677
|
+
followed by Conv1D layers. The model is compiled with the RMSprop optimizer
|
|
2678
|
+
and mean squared error loss.
|
|
2679
|
+
|
|
2680
|
+
Parameters
|
|
2681
|
+
----------
|
|
2682
|
+
self : object
|
|
2683
|
+
The instance of the class containing the model configuration attributes.
|
|
2684
|
+
|
|
2685
|
+
Attributes Used
|
|
2686
|
+
---------------
|
|
2687
|
+
self.invert : bool
|
|
2688
|
+
If True, the model begins with a Conv1D layer and ends with an LSTM layer.
|
|
2689
|
+
If False, the model begins with an LSTM layer and ends with a Conv1D layer.
|
|
2690
|
+
self.num_filters : int
|
|
2691
|
+
Number of filters in each Conv1D layer.
|
|
2692
|
+
self.kernel_size : int
|
|
2693
|
+
Size of the kernel in Conv1D layers.
|
|
2694
|
+
self.padding : str, default='same'
|
|
2695
|
+
Padding mode for Conv1D layers.
|
|
2696
|
+
self.window_size : int
|
|
2697
|
+
Length of the input sequence.
|
|
2698
|
+
self.inputsize : int
|
|
2699
|
+
Number of features in the input data.
|
|
2700
|
+
self.num_layers : int
|
|
2701
|
+
Total number of layers in the model.
|
|
2702
|
+
self.num_units : int
|
|
2703
|
+
Number of units in the LSTM layers.
|
|
2704
|
+
self.dropout_rate : float
|
|
2705
|
+
Dropout rate for regularization.
|
|
2706
|
+
self.activation : str or callable
|
|
2707
|
+
Activation function for Conv1D layers.
|
|
2708
|
+
|
|
2709
|
+
Returns
|
|
2710
|
+
-------
|
|
2711
|
+
None
|
|
2712
|
+
This method modifies the instance's `self.model` attribute in place.
|
|
2713
|
+
|
|
2714
|
+
Notes
|
|
2715
|
+
-----
|
|
2716
|
+
- The model uses `Sequential` from Keras.
|
|
2717
|
+
- Batch normalization and dropout are applied after each Conv1D layer (except the last).
|
|
2718
|
+
- The final layer is a Dense layer wrapped in `TimeDistributed` for sequence-to-sequence output.
|
|
2719
|
+
- The model is compiled using `RMSprop` optimizer and `mse` loss.
|
|
2720
|
+
|
|
2721
|
+
Examples
|
|
2722
|
+
--------
|
|
2723
|
+
>>> model_builder = MyModelClass()
|
|
2724
|
+
>>> model_builder.invert = True
|
|
2725
|
+
>>> model_builder.num_filters = 32
|
|
2726
|
+
>>> model_builder.kernel_size = 3
|
|
2727
|
+
>>> model_builder.window_size = 100
|
|
2728
|
+
>>> model_builder.inputsize = 1
|
|
2729
|
+
>>> model_builder.num_layers = 5
|
|
2730
|
+
>>> model_builder.num_units = 64
|
|
2731
|
+
>>> model_builder.dropout_rate = 0.2
|
|
2732
|
+
>>> model_builder.activation = 'relu'
|
|
2733
|
+
>>> model_builder.makenet()
|
|
2734
|
+
>>> model_builder.model.summary()
|
|
2735
|
+
"""
|
|
974
2736
|
self.model = Sequential()
|
|
975
2737
|
|
|
976
2738
|
if self.invert:
|
|
@@ -1052,14 +2814,67 @@ class HybridDLFilter(DeepLearningFilter):
|
|
|
1052
2814
|
|
|
1053
2815
|
|
|
1054
2816
|
def filtscale(
|
|
1055
|
-
data,
|
|
1056
|
-
scalefac=1.0,
|
|
1057
|
-
reverse=False,
|
|
1058
|
-
hybrid=False,
|
|
1059
|
-
lognormalize=True,
|
|
1060
|
-
epsilon=1e-10,
|
|
1061
|
-
numorders=6,
|
|
1062
|
-
):
|
|
2817
|
+
data: NDArray,
|
|
2818
|
+
scalefac: float = 1.0,
|
|
2819
|
+
reverse: bool = False,
|
|
2820
|
+
hybrid: bool = False,
|
|
2821
|
+
lognormalize: bool = True,
|
|
2822
|
+
epsilon: float = 1e-10,
|
|
2823
|
+
numorders: int = 6,
|
|
2824
|
+
) -> tuple[NDArray, float] | NDArray:
|
|
2825
|
+
"""
|
|
2826
|
+
Apply or reverse a frequency-domain scaling and normalization to input data.
|
|
2827
|
+
|
|
2828
|
+
This function performs either forward or inverse transformation of the input
|
|
2829
|
+
data in the frequency domain, applying scaling, normalization, and optionally
|
|
2830
|
+
hybrid encoding. It supports both logarithmic and standard normalization
|
|
2831
|
+
modes.
|
|
2832
|
+
|
|
2833
|
+
Parameters
|
|
2834
|
+
----------
|
|
2835
|
+
data : NDArray
|
|
2836
|
+
Input signal or transformed data (depending on `reverse` flag).
|
|
2837
|
+
scalefac : float, optional
|
|
2838
|
+
Scaling factor used for normalization. Default is 1.0.
|
|
2839
|
+
reverse : bool, optional
|
|
2840
|
+
If True, performs inverse transformation from frequency domain back
|
|
2841
|
+
to time domain. Default is False.
|
|
2842
|
+
hybrid : bool, optional
|
|
2843
|
+
If True, returns a hybrid representation combining original data
|
|
2844
|
+
and magnitude spectrum. Default is False.
|
|
2845
|
+
lognormalize : bool, optional
|
|
2846
|
+
If True, applies logarithmic normalization to the magnitude spectrum.
|
|
2847
|
+
Default is True.
|
|
2848
|
+
epsilon : float, optional
|
|
2849
|
+
Small constant added to avoid log(0). Default is 1e-10.
|
|
2850
|
+
numorders : int, optional
|
|
2851
|
+
Number of orders used for scaling in log normalization. Default is 6.
|
|
2852
|
+
|
|
2853
|
+
Returns
|
|
2854
|
+
-------
|
|
2855
|
+
tuple[NDArray, float] or NDArray
|
|
2856
|
+
- If `reverse=False`: A tuple of (transformed data, scale factor).
|
|
2857
|
+
The transformed data is a stacked array of magnitude and phase components
|
|
2858
|
+
(or original data in hybrid mode).
|
|
2859
|
+
- If `reverse=True`: Reconstructed time-domain signal.
|
|
2860
|
+
|
|
2861
|
+
Notes
|
|
2862
|
+
-----
|
|
2863
|
+
- Forward mode applies FFT, normalizes the magnitude, and stacks magnitude
|
|
2864
|
+
and phase.
|
|
2865
|
+
- In hybrid mode, the output includes the original time-domain signal
|
|
2866
|
+
instead of the phase component.
|
|
2867
|
+
- In reverse mode, the phase and magnitude components are used to reconstruct
|
|
2868
|
+
the original signal using inverse FFT.
|
|
2869
|
+
|
|
2870
|
+
Examples
|
|
2871
|
+
--------
|
|
2872
|
+
>>> import numpy as np
|
|
2873
|
+
>>> from scipy import fftpack
|
|
2874
|
+
>>> x = np.random.randn(1024)
|
|
2875
|
+
>>> scaled_data, scale = filtscale(x)
|
|
2876
|
+
>>> reconstructed = filtscale(scaled_data, scale, reverse=True)
|
|
2877
|
+
"""
|
|
1063
2878
|
if not reverse:
|
|
1064
2879
|
specvals = fftpack.fft(data)
|
|
1065
2880
|
if lognormalize:
|
|
@@ -1089,17 +2904,129 @@ def filtscale(
|
|
|
1089
2904
|
return fftpack.ifft(specvals).real
|
|
1090
2905
|
|
|
1091
2906
|
|
|
1092
|
-
def tobadpts(name):
|
|
2907
|
+
def tobadpts(name: str) -> str:
|
|
2908
|
+
"""
|
|
2909
|
+
Convert a filename to its corresponding bad points filename.
|
|
2910
|
+
|
|
2911
|
+
Replaces the '.txt' extension with '_badpts.txt' to create a standardized
|
|
2912
|
+
bad points filename pattern.
|
|
2913
|
+
|
|
2914
|
+
Parameters
|
|
2915
|
+
----------
|
|
2916
|
+
name : str
|
|
2917
|
+
Input filename, typically ending with '.txt' extension.
|
|
2918
|
+
|
|
2919
|
+
Returns
|
|
2920
|
+
-------
|
|
2921
|
+
str
|
|
2922
|
+
Filename with '.txt' replaced by '_badpts.txt' extension.
|
|
2923
|
+
|
|
2924
|
+
Notes
|
|
2925
|
+
-----
|
|
2926
|
+
This function assumes the input filename ends with '.txt' extension.
|
|
2927
|
+
If the input does not contain '.txt', the function will append '_badpts.txt'
|
|
2928
|
+
to the end of the string.
|
|
2929
|
+
|
|
2930
|
+
Examples
|
|
2931
|
+
--------
|
|
2932
|
+
>>> tobadpts("data.txt")
|
|
2933
|
+
'data_badpts.txt'
|
|
2934
|
+
|
|
2935
|
+
>>> tobadpts("results.txt")
|
|
2936
|
+
'results_badpts.txt'
|
|
2937
|
+
|
|
2938
|
+
>>> tobadpts("config")
|
|
2939
|
+
'config_badpts.txt'
|
|
2940
|
+
"""
|
|
1093
2941
|
return name.replace(".txt", "_badpts.txt")
|
|
1094
2942
|
|
|
1095
2943
|
|
|
1096
|
-
def targettoinput(name, targetfrag="xyz", inputfrag="abc"):
|
|
2944
|
+
def targettoinput(name: str, targetfrag: str = "xyz", inputfrag: str = "abc") -> str:
|
|
2945
|
+
"""
|
|
2946
|
+
Replace occurrences of a target fragment with an input fragment in a string.
|
|
2947
|
+
|
|
2948
|
+
Parameters
|
|
2949
|
+
----------
|
|
2950
|
+
name : str
|
|
2951
|
+
The input string to process.
|
|
2952
|
+
targetfrag : str, default='xyz'
|
|
2953
|
+
The fragment to search for and replace. Defaults to 'xyz'.
|
|
2954
|
+
inputfrag : str, default='abc'
|
|
2955
|
+
The fragment to replace targetfrag with. Defaults to 'abc'.
|
|
2956
|
+
|
|
2957
|
+
Returns
|
|
2958
|
+
-------
|
|
2959
|
+
str
|
|
2960
|
+
The modified string with targetfrag replaced by inputfrag.
|
|
2961
|
+
|
|
2962
|
+
Notes
|
|
2963
|
+
-----
|
|
2964
|
+
This function uses Python's built-in string replace method, which replaces
|
|
2965
|
+
all occurrences of targetfrag with inputfrag in the input string.
|
|
2966
|
+
|
|
2967
|
+
Examples
|
|
2968
|
+
--------
|
|
2969
|
+
>>> targettoinput("hello xyz world")
|
|
2970
|
+
'hello abc world'
|
|
2971
|
+
|
|
2972
|
+
>>> targettoinput("xyzxyzxyz", "xyz", "123")
|
|
2973
|
+
'123123123'
|
|
2974
|
+
|
|
2975
|
+
>>> targettoinput("abcdef", "xyz", "123")
|
|
2976
|
+
'abcdef'
|
|
2977
|
+
"""
|
|
1097
2978
|
LGR.debug(f"replacing {targetfrag} with {inputfrag}")
|
|
1098
2979
|
return name.replace(targetfrag, inputfrag)
|
|
1099
2980
|
|
|
1100
2981
|
|
|
1101
|
-
def getmatchedtcs(
|
|
1102
|
-
|
|
2982
|
+
def getmatchedtcs(
|
|
2983
|
+
searchstring: str,
|
|
2984
|
+
usebadpts: bool = False,
|
|
2985
|
+
targetfrag: str = "xyz",
|
|
2986
|
+
inputfrag: str = "abc",
|
|
2987
|
+
debug: bool = False,
|
|
2988
|
+
) -> tuple[list[str], int]:
|
|
2989
|
+
"""
|
|
2990
|
+
Find and validate timecourse files matching a search pattern, and determine the length of the timecourses.
|
|
2991
|
+
|
|
2992
|
+
This function searches for files matching the given `searchstring`, checks for the existence of
|
|
2993
|
+
corresponding info files to ensure completeness, and reads the first matched file to determine
|
|
2994
|
+
the length of the timecourses. It is intended for use with BIDS-compatible timecourse files.
|
|
2995
|
+
|
|
2996
|
+
Parameters
|
|
2997
|
+
----------
|
|
2998
|
+
searchstring : str
|
|
2999
|
+
A glob pattern to match target files. Typically includes a path and file name pattern.
|
|
3000
|
+
usebadpts : bool, optional
|
|
3001
|
+
Flag indicating whether to use bad points in processing. Default is False.
|
|
3002
|
+
targetfrag : str, optional
|
|
3003
|
+
Fragment identifier for target files. Default is "xyz".
|
|
3004
|
+
inputfrag : str, optional
|
|
3005
|
+
Fragment identifier for input files. Default is "abc".
|
|
3006
|
+
debug : bool, optional
|
|
3007
|
+
If True, print debug information including matched files and processing steps.
|
|
3008
|
+
Default is False.
|
|
3009
|
+
|
|
3010
|
+
Returns
|
|
3011
|
+
-------
|
|
3012
|
+
tuple[list[str], int]
|
|
3013
|
+
A tuple containing:
|
|
3014
|
+
- List of matched and validated file paths.
|
|
3015
|
+
- Length of the timecourses (number of timepoints) in the first matched file.
|
|
3016
|
+
|
|
3017
|
+
Notes
|
|
3018
|
+
-----
|
|
3019
|
+
- The function expects files to have a corresponding `_info` file for validation.
|
|
3020
|
+
- Timecourse data is read from the first matched file using `tide_io.readbidstsv`.
|
|
3021
|
+
- The function currently only reads the first matched file to determine `tclen`, assuming all
|
|
3022
|
+
matched files have the same timecourse length.
|
|
3023
|
+
|
|
3024
|
+
Examples
|
|
3025
|
+
--------
|
|
3026
|
+
>>> matched_files, length = getmatchedtcs("sub-*/func/*cardiacfromfmri_25.0Hz*")
|
|
3027
|
+
>>> print(f"Found {len(matched_files)} files with {length} timepoints each.")
|
|
3028
|
+
"""
|
|
3029
|
+
# list all the target files
|
|
1103
3030
|
fromfile = sorted(glob.glob(searchstring))
|
|
1104
3031
|
if debug:
|
|
1105
3032
|
print(f"searchstring: {searchstring} -> {fromfile}")
|
|
@@ -1136,18 +3063,142 @@ def getmatchedtcs(searchstring, usebadpts=False, targetfrag="xyz", inputfrag="ab
|
|
|
1136
3063
|
|
|
1137
3064
|
|
|
1138
3065
|
def readindata(
|
|
1139
|
-
matchedfilelist,
|
|
1140
|
-
tclen,
|
|
1141
|
-
targetfrag="xyz",
|
|
1142
|
-
inputfrag="abc",
|
|
1143
|
-
usebadpts=False,
|
|
1144
|
-
startskip=0,
|
|
1145
|
-
endskip=0,
|
|
1146
|
-
corrthresh=0.5,
|
|
1147
|
-
readlim=None,
|
|
1148
|
-
readskip=None,
|
|
1149
|
-
debug=False,
|
|
1150
|
-
):
|
|
3066
|
+
matchedfilelist: list[str],
|
|
3067
|
+
tclen: int,
|
|
3068
|
+
targetfrag: str = "xyz",
|
|
3069
|
+
inputfrag: str = "abc",
|
|
3070
|
+
usebadpts: bool = False,
|
|
3071
|
+
startskip: int = 0,
|
|
3072
|
+
endskip: int = 0,
|
|
3073
|
+
corrthresh: float = 0.5,
|
|
3074
|
+
readlim: int | None = None,
|
|
3075
|
+
readskip: int | None = None,
|
|
3076
|
+
debug: bool = False,
|
|
3077
|
+
) -> tuple[NDArray, NDArray, list[str]] | tuple[NDArray, NDArray, list[str], NDArray]:
|
|
3078
|
+
"""
|
|
3079
|
+
Read and process time-series data from a list of matched files.
|
|
3080
|
+
|
|
3081
|
+
This function reads cardiac and plethysmographic data from a set of files, applies
|
|
3082
|
+
filtering based on data quality metrics (e.g., correlation thresholds, NaN values,
|
|
3083
|
+
data length, and standard deviation), and returns processed arrays for training.
|
|
3084
|
+
|
|
3085
|
+
Parameters
|
|
3086
|
+
----------
|
|
3087
|
+
matchedfilelist : list of str
|
|
3088
|
+
List of file paths to be processed. Each file should contain time-series data
|
|
3089
|
+
in a format compatible with `tide_io.readbidstsv`.
|
|
3090
|
+
tclen : int
|
|
3091
|
+
The length of the time series to be read from each file.
|
|
3092
|
+
targetfrag : str, optional
|
|
3093
|
+
Fragment identifier used for mapping filenames (default is "xyz").
|
|
3094
|
+
inputfrag : str, optional
|
|
3095
|
+
Fragment identifier used for mapping filenames (default is "abc").
|
|
3096
|
+
usebadpts : bool, optional
|
|
3097
|
+
If True, include bad point data in the output (default is False).
|
|
3098
|
+
startskip : int, optional
|
|
3099
|
+
Number of samples to skip at the beginning of each time series (default is 0).
|
|
3100
|
+
endskip : int, optional
|
|
3101
|
+
Number of samples to skip at the end of each time series (default is 0).
|
|
3102
|
+
corrthresh : float, optional
|
|
3103
|
+
Threshold for correlation between raw and plethysmographic signals.
|
|
3104
|
+
Files with correlations below this value are excluded (default is 0.5).
|
|
3105
|
+
readlim : int, optional
|
|
3106
|
+
Maximum number of files to read. If None, all files are read (default is None).
|
|
3107
|
+
readskip : int, optional
|
|
3108
|
+
Number of files to skip at the beginning of the list. If None, no files are skipped (default is None).
|
|
3109
|
+
debug : bool, optional
|
|
3110
|
+
If True, print debug information during processing (default is False).
|
|
3111
|
+
|
|
3112
|
+
Returns
|
|
3113
|
+
-------
|
|
3114
|
+
tuple of (NDArray, NDArray, list[str]) or (NDArray, NDArray, list[str], NDArray)
|
|
3115
|
+
- `x1[startskip:-endskip, :count]`: Array of x-axis time series data.
|
|
3116
|
+
- `y1[startskip:-endskip, :count]`: Array of y-axis time series data.
|
|
3117
|
+
- `names[:count]`: List of file names that passed quality checks.
|
|
3118
|
+
- `bad1[startskip:-endskip, :count]`: Optional array of bad point data if `usebadpts` is True.
|
|
3119
|
+
|
|
3120
|
+
Notes
|
|
3121
|
+
-----
|
|
3122
|
+
- Files are filtered based on:
|
|
3123
|
+
- Correlation threshold (`corrthresh`)
|
|
3124
|
+
- Presence of NaN values
|
|
3125
|
+
- Data length (must be at least `tclen`)
|
|
3126
|
+
- Standard deviation of data (must be between 0.5 and 20.0)
|
|
3127
|
+
- Excluded files are logged with reasons.
|
|
3128
|
+
- If `usebadpts` is True, bad point data is included in the returned tuple.
|
|
3129
|
+
|
|
3130
|
+
Examples
|
|
3131
|
+
--------
|
|
3132
|
+
>>> x, y, names = readindata(
|
|
3133
|
+
... matchedfilelist=["file1.tsv", "file2.tsv"],
|
|
3134
|
+
... tclen=1000,
|
|
3135
|
+
... corrthresh=0.6,
|
|
3136
|
+
... readlim=10
|
|
3137
|
+
... )
|
|
3138
|
+
>>> print(f"Loaded {len(names)} files")
|
|
3139
|
+
"""
|
|
3140
|
+
"""
|
|
3141
|
+
Read and process time-series data from a list of matched files.
|
|
3142
|
+
|
|
3143
|
+
This function reads cardiac and plethysmographic data from a set of files, applies
|
|
3144
|
+
filtering based on data quality metrics (e.g., correlation thresholds, NaN values,
|
|
3145
|
+
data length, and standard deviation), and returns processed arrays for training.
|
|
3146
|
+
|
|
3147
|
+
Parameters
|
|
3148
|
+
----------
|
|
3149
|
+
matchedfilelist : list of str
|
|
3150
|
+
List of file paths to be processed. Each file should contain time-series data
|
|
3151
|
+
in a format compatible with `tide_io.readbidstsv`.
|
|
3152
|
+
tclen : int
|
|
3153
|
+
The length of the time series to be read from each file.
|
|
3154
|
+
targetfrag : str, optional
|
|
3155
|
+
Fragment identifier used for mapping filenames (default is "xyz").
|
|
3156
|
+
inputfrag : str, optional
|
|
3157
|
+
Fragment identifier used for mapping filenames (default is "abc").
|
|
3158
|
+
usebadpts : bool, optional
|
|
3159
|
+
If True, include bad point data in the output (default is False).
|
|
3160
|
+
startskip : int, optional
|
|
3161
|
+
Number of samples to skip at the beginning of each time series (default is 0).
|
|
3162
|
+
endskip : int, optional
|
|
3163
|
+
Number of samples to skip at the end of each time series (default is 0).
|
|
3164
|
+
corrthresh : float, optional
|
|
3165
|
+
Threshold for correlation between raw and plethysmographic signals.
|
|
3166
|
+
Files with correlations below this value are excluded (default is 0.5).
|
|
3167
|
+
readlim : int, optional
|
|
3168
|
+
Maximum number of files to read. If None, all files are read (default is None).
|
|
3169
|
+
readskip : int, optional
|
|
3170
|
+
Number of files to skip at the beginning of the list. If None, no files are skipped (default is None).
|
|
3171
|
+
debug : bool, optional
|
|
3172
|
+
If True, print debug information during processing (default is False).
|
|
3173
|
+
|
|
3174
|
+
Returns
|
|
3175
|
+
-------
|
|
3176
|
+
tuple of (NDArray, NDArray, list[str]) or (NDArray, NDArray, list[str], NDArray)
|
|
3177
|
+
- `x1[startskip:-endskip, :count]`: Array of x-axis time series data.
|
|
3178
|
+
- `y1[startskip:-endskip, :count]`: Array of y-axis time series data.
|
|
3179
|
+
- `names[:count]`: List of file names that passed quality checks.
|
|
3180
|
+
- `bad1[startskip:-endskip, :count]`: Optional array of bad point data if `usebadpts` is True.
|
|
3181
|
+
|
|
3182
|
+
Notes
|
|
3183
|
+
-----
|
|
3184
|
+
- Files are filtered based on:
|
|
3185
|
+
- Correlation threshold (`corrthresh`)
|
|
3186
|
+
- Presence of NaN values
|
|
3187
|
+
- Data length (must be at least `tclen`)
|
|
3188
|
+
- Standard deviation of data (must be between 0.5 and 20.0)
|
|
3189
|
+
- Excluded files are logged with reasons.
|
|
3190
|
+
- If `usebadpts` is True, bad point data is included in the returned tuple.
|
|
3191
|
+
|
|
3192
|
+
Examples
|
|
3193
|
+
--------
|
|
3194
|
+
>>> x, y, names = readindata(
|
|
3195
|
+
... matchedfilelist=["file1.tsv", "file2.tsv"],
|
|
3196
|
+
... tclen=1000,
|
|
3197
|
+
... corrthresh=0.6,
|
|
3198
|
+
... readlim=10
|
|
3199
|
+
... )
|
|
3200
|
+
>>> print(f"Loaded {len(names)} files")
|
|
3201
|
+
"""
|
|
1151
3202
|
LGR.info(
|
|
1152
3203
|
"readindata called with usebadpts, startskip, endskip, readlim, readskip, targetfrag, inputfrag = "
|
|
1153
3204
|
f"{usebadpts} {startskip} {endskip} {readlim} {readskip} {targetfrag} {inputfrag}"
|
|
@@ -1296,52 +3347,117 @@ def readindata(
|
|
|
1296
3347
|
|
|
1297
3348
|
|
|
1298
3349
|
def prep(
|
|
1299
|
-
window_size,
|
|
1300
|
-
step=1,
|
|
1301
|
-
excludethresh=4.0,
|
|
1302
|
-
usebadpts=False,
|
|
1303
|
-
startskip=200,
|
|
1304
|
-
endskip=200,
|
|
1305
|
-
excludebysubject=True,
|
|
1306
|
-
thesuffix="sliceres",
|
|
1307
|
-
thedatadir="/data/frederic/physioconn/output_2025",
|
|
1308
|
-
inputfrag="abc",
|
|
1309
|
-
targetfrag="xyz",
|
|
1310
|
-
corrthresh=0.5,
|
|
1311
|
-
dofft=False,
|
|
1312
|
-
readlim=None,
|
|
1313
|
-
readskip=None,
|
|
1314
|
-
countlim=None,
|
|
1315
|
-
debug=False,
|
|
3350
|
+
window_size: int,
|
|
3351
|
+
step: int = 1,
|
|
3352
|
+
excludethresh: float = 4.0,
|
|
3353
|
+
usebadpts: bool = False,
|
|
3354
|
+
startskip: int = 200,
|
|
3355
|
+
endskip: int = 200,
|
|
3356
|
+
excludebysubject: bool = True,
|
|
3357
|
+
thesuffix: str = "sliceres",
|
|
3358
|
+
thedatadir: str = "/data/frederic/physioconn/output_2025",
|
|
3359
|
+
inputfrag: str = "abc",
|
|
3360
|
+
targetfrag: str = "xyz",
|
|
3361
|
+
corrthresh: float = 0.5,
|
|
3362
|
+
dofft: bool = False,
|
|
3363
|
+
readlim: int | None = None,
|
|
3364
|
+
readskip: int | None = None,
|
|
3365
|
+
countlim: int | None = None,
|
|
3366
|
+
debug: bool = False,
|
|
3367
|
+
) -> (
|
|
3368
|
+
tuple[NDArray, NDArray, NDArray, NDArray, int, int, int]
|
|
3369
|
+
| tuple[NDArray, NDArray, NDArray, NDArray, int, int, int, NDArray, NDArray]
|
|
1316
3370
|
):
|
|
1317
3371
|
"""
|
|
1318
|
-
|
|
3372
|
+
Prepare time-series data for training and validation by reading, normalizing,
|
|
3373
|
+
windowing, and splitting into batches.
|
|
3374
|
+
|
|
3375
|
+
This function reads time-series data from JSON files, normalizes the data,
|
|
3376
|
+
applies windowing to create input-output pairs, and splits the data into
|
|
3377
|
+
training and validation sets based on subject-wise or window-wise exclusion
|
|
3378
|
+
criteria.
|
|
1319
3379
|
|
|
1320
3380
|
Parameters
|
|
1321
3381
|
----------
|
|
1322
|
-
window_size
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
3382
|
+
window_size : int
|
|
3383
|
+
Size of the sliding window used to create input-output pairs.
|
|
3384
|
+
step : int, optional
|
|
3385
|
+
Step size for sliding window (default is 1).
|
|
3386
|
+
excludethresh : float, optional
|
|
3387
|
+
Threshold for excluding windows or subjects based on maximum absolute
|
|
3388
|
+
value of input data (default is 4.0).
|
|
3389
|
+
usebadpts : bool, optional
|
|
3390
|
+
Whether to include bad points in the data (default is False).
|
|
3391
|
+
startskip : int, optional
|
|
3392
|
+
Number of time points to skip at the beginning of each time series
|
|
3393
|
+
(default is 200).
|
|
3394
|
+
endskip : int, optional
|
|
3395
|
+
Number of time points to skip at the end of each time series
|
|
3396
|
+
(default is 200).
|
|
3397
|
+
excludebysubject : bool, optional
|
|
3398
|
+
If True, exclude entire subjects based on maximum absolute value;
|
|
3399
|
+
otherwise, exclude individual windows (default is True).
|
|
3400
|
+
thesuffix : str, optional
|
|
3401
|
+
Suffix used to identify files (default is "sliceres").
|
|
3402
|
+
thedatadir : str, optional
|
|
3403
|
+
Directory where the data files are located (default is
|
|
3404
|
+
"/data/frederic/physioconn/output_2025").
|
|
3405
|
+
inputfrag : str, optional
|
|
3406
|
+
Fragment identifier for input data (default is "abc").
|
|
3407
|
+
targetfrag : str, optional
|
|
3408
|
+
Fragment identifier for target data (default is "xyz").
|
|
3409
|
+
corrthresh : float, optional
|
|
3410
|
+
Correlation threshold for matching time series (default is 0.5).
|
|
3411
|
+
dofft : bool, optional
|
|
3412
|
+
Whether to perform FFT on the data (default is False).
|
|
3413
|
+
readlim : int or None, optional
|
|
3414
|
+
Limit on number of time points to read (default is None).
|
|
3415
|
+
readskip : int or None, optional
|
|
3416
|
+
Number of time points to skip when reading data (default is None).
|
|
3417
|
+
countlim : int or None, optional
|
|
3418
|
+
Limit on number of subjects to process (default is None).
|
|
3419
|
+
debug : bool, optional
|
|
3420
|
+
Whether to enable debug logging (default is False).
|
|
1338
3421
|
|
|
1339
3422
|
Returns
|
|
1340
3423
|
-------
|
|
1341
|
-
|
|
1342
|
-
|
|
3424
|
+
tuple of (NDArray, NDArray, NDArray, NDArray, int, int, int)
|
|
3425
|
+
If `dofft` is False:
|
|
3426
|
+
- train_x : ndarray of shape (n_train, window_size, 1)
|
|
3427
|
+
- train_y : ndarray of shape (n_train, window_size, 1)
|
|
3428
|
+
- val_x : ndarray of shape (n_val, window_size, 1)
|
|
3429
|
+
- val_y : ndarray of shape (n_val, window_size, 1)
|
|
3430
|
+
- N_subjs : int
|
|
3431
|
+
- tclen : int
|
|
3432
|
+
- batchsize : int
|
|
3433
|
+
|
|
3434
|
+
tuple of (NDArray, NDArray, NDArray, NDArray, int, int, int,
|
|
3435
|
+
NDArray, NDArray)
|
|
3436
|
+
If `dofft` is True:
|
|
3437
|
+
- train_x : ndarray of shape (n_train, window_size, 2)
|
|
3438
|
+
- train_y : ndarray of shape (n_train, window_size, 2)
|
|
3439
|
+
- val_x : ndarray of shape (n_val, window_size, 2)
|
|
3440
|
+
- val_y : ndarray of shape (n_val, window_size, 2)
|
|
3441
|
+
- N_subjs : int
|
|
3442
|
+
- tclen : int
|
|
3443
|
+
- batchsize : int
|
|
3444
|
+
- Xscale_fourier : ndarray of shape (N_subjs, windowspersubject)
|
|
3445
|
+
- Yscale_fourier : ndarray of shape (N_subjs, windowspersubject)
|
|
3446
|
+
|
|
3447
|
+
Notes
|
|
3448
|
+
-----
|
|
3449
|
+
- Data normalization is performed using median absolute deviation (MAD).
|
|
3450
|
+
- Windows are created based on sliding window approach.
|
|
3451
|
+
- Training and validation sets are split based on subject-wise partitioning.
|
|
3452
|
+
- If `usebadpts` is True, bad points are included in the returned data.
|
|
3453
|
+
- If `dofft` is True, data is transformed using a filtering scale function.
|
|
3454
|
+
|
|
3455
|
+
Examples
|
|
3456
|
+
--------
|
|
3457
|
+
>>> train_x, train_y, val_x, val_y, N_subjs, tclen, batchsize = prep(
|
|
3458
|
+
... window_size=100, step=10, excludethresh=3.0, dofft=False
|
|
3459
|
+
... )
|
|
1343
3460
|
"""
|
|
1344
|
-
|
|
1345
3461
|
searchstring = os.path.join(thedatadir, "*", "*_desc-stdrescardfromfmri_timeseries.json")
|
|
1346
3462
|
|
|
1347
3463
|
# find matched files
|