flowml 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowml/__init__.py +19 -0
- flowml/models/__init__.py +14 -0
- flowml/models/base.py +89 -0
- flowml/models/pytorch/__init__.py +9 -0
- flowml/models/pytorch/architectures/lstm.py +74 -0
- flowml/models/pytorch/architectures/mlp.py +45 -0
- flowml/models/pytorch/architectures/transformer.py +137 -0
- flowml/models/pytorch/base.py +272 -0
- flowml/models/pytorch/drivers/lstm_autoregressive.py +220 -0
- flowml/models/pytorch/drivers/lstm_direct_regressor.py +180 -0
- flowml/models/pytorch/drivers/mlp_regressor.py +180 -0
- flowml/models/pytorch/drivers/transformer_regressor.py +192 -0
- flowml/models/pytorch/utils.py +175 -0
- flowml/models/xgboost/__init__.py +7 -0
- flowml/models/xgboost/regressor.py +91 -0
- flowml/pipefill.py +160 -0
- flowml/tensorboard.py +44 -0
- flowml/utils.py +71 -0
- flowml-0.1.0.dist-info/LICENSE +201 -0
- flowml-0.1.0.dist-info/METADATA +214 -0
- flowml-0.1.0.dist-info/RECORD +22 -0
- flowml-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from typing import Any
|
|
3
|
+
import numpy.typing as npt
|
|
4
|
+
from flowml.models.pytorch.base import BasePyTorchModel
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from flowdapt.lib.logger import get_logger
|
|
7
|
+
from torch.optim.lr_scheduler import ReduceLROnPlateau
|
|
8
|
+
from flowml.models.pytorch.utils import EarlyStopping, WindowAutoRegressiveDataset
|
|
9
|
+
from flowml.models.pytorch.architectures.lstm import LSTMAutoRegressive
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
logger = get_logger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class PyTorchLSTMAutoregressive(BasePyTorchModel):
|
|
16
|
+
|
|
17
|
+
def fit(self,
|
|
18
|
+
xs: npt.ArrayLike | pd.DataFrame,
|
|
19
|
+
ys: npt.ArrayLike | pd.DataFrame,
|
|
20
|
+
eval_set: Any = None):
|
|
21
|
+
|
|
22
|
+
x, y, valid_x, valid_y = self._validate_args(xs, ys, eval_set)
|
|
23
|
+
|
|
24
|
+
if x is None or y is None:
|
|
25
|
+
logger.warning("No training data provided, skipping fit")
|
|
26
|
+
|
|
27
|
+
dataset = self.make_dataloader(x, y, self.lookback)
|
|
28
|
+
if valid_x is not None:
|
|
29
|
+
valid_dataset = self.make_dataloader(valid_x, valid_y, self.lookback, batch_size=1, valid=True)
|
|
30
|
+
else:
|
|
31
|
+
valid_dataset = None
|
|
32
|
+
|
|
33
|
+
input_dim = x.shape[1]
|
|
34
|
+
output_dim = x.shape[1]
|
|
35
|
+
# output_dim = ys.shape[1]
|
|
36
|
+
self.nn = LSTMAutoRegressive(
|
|
37
|
+
input_features=input_dim,
|
|
38
|
+
hidden_dim=self.hidden_dim,
|
|
39
|
+
output_features=output_dim,
|
|
40
|
+
lookback=self.lookback
|
|
41
|
+
).to(self._device)
|
|
42
|
+
|
|
43
|
+
self.nn, self.optimizer = self.train_loop(dataset, valid_dataset, self.nn)
|
|
44
|
+
return self
|
|
45
|
+
|
|
46
|
+
def predict(self, xs: npt.ArrayLike):
|
|
47
|
+
x, _, _, _ = self._validate_args(xs)
|
|
48
|
+
logger.info(f"Predicting on {x.shape[0]} samples and {x.shape[1]} features")
|
|
49
|
+
|
|
50
|
+
self.nn.to(self._device)
|
|
51
|
+
self.nn.eval()
|
|
52
|
+
data = x.unsqueeze(0)
|
|
53
|
+
data = data.to(self._device)
|
|
54
|
+
preds, _, _ = self.get_rollout_pred(data, self.nn, hn=self.hn, cn=self.cn)
|
|
55
|
+
preds = preds.cpu()
|
|
56
|
+
preds = preds.squeeze()
|
|
57
|
+
return preds.detach().numpy()
|
|
58
|
+
|
|
59
|
+
def make_dataloader(self,
|
|
60
|
+
xs: torch.Tensor,
|
|
61
|
+
ys: torch.Tensor,
|
|
62
|
+
lookback: int = 10,
|
|
63
|
+
batch_size: int | None = None,
|
|
64
|
+
valid: bool = False):
|
|
65
|
+
|
|
66
|
+
dataset = WindowAutoRegressiveDataset(
|
|
67
|
+
xs, window_size=lookback, rollout=6, valid=valid)
|
|
68
|
+
|
|
69
|
+
if batch_size is None:
|
|
70
|
+
batch_size = self.batch_size
|
|
71
|
+
|
|
72
|
+
dataloader = torch.utils.data.DataLoader(
|
|
73
|
+
dataset,
|
|
74
|
+
batch_size=batch_size,
|
|
75
|
+
drop_last=False,
|
|
76
|
+
num_workers=0,
|
|
77
|
+
shuffle=self.shuffle
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
return dataloader
|
|
81
|
+
|
|
82
|
+
def train_loop(self, dataloader: Any, valid_dataloader: Any, model: torch.nn.Module):
|
|
83
|
+
|
|
84
|
+
model.train()
|
|
85
|
+
criterion = torch.nn.MSELoss()
|
|
86
|
+
optimizer = torch.optim.Adam(
|
|
87
|
+
model.parameters(), lr=1e-4, weight_decay=1e-4)
|
|
88
|
+
learning_rate_scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.5, patience=5)
|
|
89
|
+
early_stopping = EarlyStopping(patience=50)
|
|
90
|
+
|
|
91
|
+
logger.info("Starting training loop")
|
|
92
|
+
min_valid_loss = 1e10
|
|
93
|
+
total_batches = 0
|
|
94
|
+
best_model = None
|
|
95
|
+
best_optim = None
|
|
96
|
+
|
|
97
|
+
for epoch in range(0, self.epochs):
|
|
98
|
+
train_loss = 0.
|
|
99
|
+
last_batch_time = time.time()
|
|
100
|
+
# hn, cn = None, None
|
|
101
|
+
# first = True
|
|
102
|
+
for batch, batch_data in enumerate(dataloader):
|
|
103
|
+
x, y_target = batch_data
|
|
104
|
+
x = x.float().to(self._device)
|
|
105
|
+
# if not first:
|
|
106
|
+
# hn = hn.expand(-1, x.shape[0], -1).contiguous()
|
|
107
|
+
# cn = cn.expand(-1, x.shape[0], -1).contiguous()
|
|
108
|
+
y_target = y_target.float().to(self._device)
|
|
109
|
+
loss, _, _, _ = self.get_rollout_loss(x, y_target, model, criterion)
|
|
110
|
+
# Backprogation
|
|
111
|
+
optimizer.zero_grad()
|
|
112
|
+
loss.backward()
|
|
113
|
+
optimizer.step()
|
|
114
|
+
total_batches += 1
|
|
115
|
+
train_loss += loss.item()
|
|
116
|
+
self.log("train_loss", loss.item() /
|
|
117
|
+
(self.batch_size * 6), total_batches)
|
|
118
|
+
total_batches += 1
|
|
119
|
+
|
|
120
|
+
# # normalize the hidden states across the batch
|
|
121
|
+
# hn = torch.mean(hn.detach(), dim=1).unsqueeze(1)
|
|
122
|
+
# cn = torch.mean(cn.detach(), dim=1).unsqueeze(1)
|
|
123
|
+
# first = False
|
|
124
|
+
|
|
125
|
+
samples_per_second = self.batch_size / (time.time() - last_batch_time)
|
|
126
|
+
self.log("samples_per_second", samples_per_second, epoch)
|
|
127
|
+
|
|
128
|
+
valid_loss = 0.
|
|
129
|
+
if valid_dataloader is not None:
|
|
130
|
+
# get loss on valid_dataset
|
|
131
|
+
model.eval()
|
|
132
|
+
# hn_v = None # hn.detach()
|
|
133
|
+
# cn_v = None # cn.detach()
|
|
134
|
+
with torch.no_grad():
|
|
135
|
+
for batch, batch_data in enumerate(valid_dataloader):
|
|
136
|
+
x, y_target = batch_data
|
|
137
|
+
x = x.float().to(self._device)
|
|
138
|
+
y_target = y_target.float().to(self._device)
|
|
139
|
+
loss, _, _, _ = self.get_rollout_loss(x, y_target, model, criterion)
|
|
140
|
+
# , hn=hn_v, cn=cn_v)
|
|
141
|
+
valid_loss += loss.item()
|
|
142
|
+
|
|
143
|
+
valid_loss /= (float(batch+1) * self.batch_size * 6) # 6 is the rollout
|
|
144
|
+
model.train()
|
|
145
|
+
learning_rate_scheduler.step(valid_loss)
|
|
146
|
+
|
|
147
|
+
# save best model for later return
|
|
148
|
+
if valid_loss < min_valid_loss:
|
|
149
|
+
logger.debug(f"found best model with loss {valid_loss}")
|
|
150
|
+
best_model = model
|
|
151
|
+
best_optim = optimizer
|
|
152
|
+
min_valid_loss = valid_loss
|
|
153
|
+
self.hn = None # hn_v.detach()
|
|
154
|
+
self.cn = None # cn_v.detach()
|
|
155
|
+
self.best_loss = float(min_valid_loss)
|
|
156
|
+
|
|
157
|
+
if early_stopping(valid_loss) and self.early_stopping:
|
|
158
|
+
logger.debug("Model is not improving, Early stopping activated")
|
|
159
|
+
break
|
|
160
|
+
|
|
161
|
+
self.log("valid_loss", valid_loss, step=epoch)
|
|
162
|
+
|
|
163
|
+
lrs = []
|
|
164
|
+
for grp in optimizer.param_groups:
|
|
165
|
+
lrs.append(grp["lr"])
|
|
166
|
+
self.log("lr", lrs[0], epoch)
|
|
167
|
+
|
|
168
|
+
logger.debug(f"Epoch {epoch}/{self.epochs}: Loss/train "
|
|
169
|
+
f"{train_loss} Loss/valid {valid_loss}")
|
|
170
|
+
|
|
171
|
+
self.reset_logger()
|
|
172
|
+
if not best_model:
|
|
173
|
+
best_model = model
|
|
174
|
+
best_optim = optimizer
|
|
175
|
+
return best_model, best_optim
|
|
176
|
+
|
|
177
|
+
def get_rollout_loss(self, x, y_target, model, criterion, hn=None, cn=None):
|
|
178
|
+
"""
|
|
179
|
+
Given the X and y_target, compute predictions and loss for each step
|
|
180
|
+
along the desired rollout. Collect full loss during rollout and return
|
|
181
|
+
loss to the optimizer.
|
|
182
|
+
"""
|
|
183
|
+
logger.debug(f"x, y shape {x.shape}, {y_target.shape}")
|
|
184
|
+
|
|
185
|
+
batch_size = x.shape[0]
|
|
186
|
+
rollout = y_target.shape[1]
|
|
187
|
+
# loop through the columns of y_target at intervals of self.lookback
|
|
188
|
+
# for t in np.arange(0, y_target.shape[1], self.output_dim):
|
|
189
|
+
for t in range(0, rollout):
|
|
190
|
+
y = y_target[:, t, :]
|
|
191
|
+
|
|
192
|
+
im, hn, cn = model(x, hn, cn)
|
|
193
|
+
if t == 0:
|
|
194
|
+
loss = criterion(
|
|
195
|
+
im.reshape(batch_size, -1), y.reshape(batch_size, -1)
|
|
196
|
+
)
|
|
197
|
+
pred = im
|
|
198
|
+
else:
|
|
199
|
+
loss += criterion(
|
|
200
|
+
im.reshape(batch_size, -1), y.reshape(batch_size, -1)
|
|
201
|
+
)
|
|
202
|
+
pred = torch.cat((pred, im), dim=1)
|
|
203
|
+
x = torch.cat((x[:, 1:, :], im), dim=1)
|
|
204
|
+
|
|
205
|
+
return loss, pred, hn, cn
|
|
206
|
+
|
|
207
|
+
def get_rollout_pred(self, x, model, rollout=6, hn=None, cn=None):
|
|
208
|
+
"""
|
|
209
|
+
Given data to inference, x, compute the preidictions for a given
|
|
210
|
+
rollout. Return the predictions
|
|
211
|
+
"""
|
|
212
|
+
for t in range(0, rollout):
|
|
213
|
+
im, hn, cn = model(x, hn, cn)
|
|
214
|
+
if t == 0:
|
|
215
|
+
pred = im
|
|
216
|
+
else:
|
|
217
|
+
pred = torch.cat((pred, im), dim=1)
|
|
218
|
+
x = torch.cat((x[:, 1:, :], im), dim=1)
|
|
219
|
+
|
|
220
|
+
return pred, hn, cn
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from typing import Any
|
|
3
|
+
import numpy.typing as npt
|
|
4
|
+
from flowml.models.pytorch.base import BasePyTorchModel
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from flowdapt.lib.logger import get_logger
|
|
7
|
+
from torch.optim.lr_scheduler import ReduceLROnPlateau
|
|
8
|
+
from flowml.models.pytorch.utils import EarlyStopping, WindowDatasetTransformer
|
|
9
|
+
from flowml.models.pytorch.architectures.lstm import LSTMDirect
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
logger = get_logger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class PyTorchLSTMDirect(BasePyTorchModel):
|
|
16
|
+
|
|
17
|
+
def fit(self,
|
|
18
|
+
xs: npt.ArrayLike | pd.DataFrame,
|
|
19
|
+
ys: npt.ArrayLike | pd.DataFrame,
|
|
20
|
+
eval_set: Any = None):
|
|
21
|
+
|
|
22
|
+
x, y, valid_x, valid_y = self._validate_args(xs, ys, eval_set)
|
|
23
|
+
|
|
24
|
+
if x is None or y is None:
|
|
25
|
+
logger.warning("No training data provided, skipping fit")
|
|
26
|
+
|
|
27
|
+
dataset = self.make_dataloader(x, y, self.lookback)
|
|
28
|
+
if valid_x is not None:
|
|
29
|
+
valid_dataset = self.make_dataloader(valid_x, valid_y, self.lookback, batch_size=1)
|
|
30
|
+
else:
|
|
31
|
+
valid_dataset = None
|
|
32
|
+
|
|
33
|
+
input_dim = x.shape[1]
|
|
34
|
+
output_dim = y.shape[1]
|
|
35
|
+
self.nn = LSTMDirect(
|
|
36
|
+
input_features=input_dim,
|
|
37
|
+
hidden_dim=self.hidden_dim,
|
|
38
|
+
output_features=output_dim,
|
|
39
|
+
lookback=self.lookback).to(self._device)
|
|
40
|
+
self.nn, self.optimizer = self.train_loop(dataset, valid_dataset, self.nn)
|
|
41
|
+
|
|
42
|
+
return self
|
|
43
|
+
|
|
44
|
+
def predict(self, xs: npt.ArrayLike):
|
|
45
|
+
x, _, _, _ = self._validate_args(xs)
|
|
46
|
+
# logger.info(f"Predicting on {x.shape[0]} samples and {x.shape[1]} features")
|
|
47
|
+
|
|
48
|
+
# self.nn.to(self._device)
|
|
49
|
+
self.nn.eval()
|
|
50
|
+
data = x.unsqueeze(0)
|
|
51
|
+
data = data.to(self._device)
|
|
52
|
+
preds, _, _ = self.nn(data, self.lookback)
|
|
53
|
+
preds = preds.cpu()
|
|
54
|
+
preds = preds.squeeze()
|
|
55
|
+
return preds.detach().numpy()
|
|
56
|
+
|
|
57
|
+
def make_dataloader(self,
|
|
58
|
+
xs: torch.Tensor,
|
|
59
|
+
ys: torch.Tensor,
|
|
60
|
+
lookback: int = 10,
|
|
61
|
+
batch_size: int | None = None):
|
|
62
|
+
|
|
63
|
+
if batch_size is None:
|
|
64
|
+
batch_size = self.batch_size
|
|
65
|
+
|
|
66
|
+
dataset = WindowDatasetTransformer(xs,
|
|
67
|
+
ys,
|
|
68
|
+
window_size=lookback)
|
|
69
|
+
|
|
70
|
+
dataloader = torch.utils.data.DataLoader(
|
|
71
|
+
dataset,
|
|
72
|
+
batch_size=batch_size,
|
|
73
|
+
drop_last=True,
|
|
74
|
+
num_workers=0,
|
|
75
|
+
shuffle=self.shuffle
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
return dataloader
|
|
79
|
+
|
|
80
|
+
def train_loop(self, dataloader: Any, valid_dataloader: Any, model: torch.nn.Module):
|
|
81
|
+
|
|
82
|
+
model.train()
|
|
83
|
+
criterion = torch.nn.MSELoss()
|
|
84
|
+
optimizer = torch.optim.Adam(
|
|
85
|
+
model.parameters(), lr=1e-4, weight_decay=1e-4)
|
|
86
|
+
learning_rate_scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.5, patience=5)
|
|
87
|
+
early_stopping = EarlyStopping(patience=20)
|
|
88
|
+
|
|
89
|
+
logger.info("Starting training loop")
|
|
90
|
+
min_valid_loss = 1e10
|
|
91
|
+
total_batches = 0
|
|
92
|
+
for epoch in range(0, self.epochs):
|
|
93
|
+
train_loss = 0.
|
|
94
|
+
last_batch_time = time.time()
|
|
95
|
+
for batch, batch_data in enumerate(dataloader):
|
|
96
|
+
x, y_target = batch_data
|
|
97
|
+
x = x.float().to(self._device) # type: ignore
|
|
98
|
+
y_target = y_target.float().to(self._device)
|
|
99
|
+
y_pred, _, _ = model(x, self.lookback)
|
|
100
|
+
loss = criterion(y_pred, y_target)
|
|
101
|
+
# Backprogation
|
|
102
|
+
optimizer.zero_grad()
|
|
103
|
+
loss.backward()
|
|
104
|
+
optimizer.step()
|
|
105
|
+
total_batches += 1
|
|
106
|
+
train_loss += loss.item()
|
|
107
|
+
self.log("train_loss", loss.item(), total_batches)
|
|
108
|
+
total_batches += 1
|
|
109
|
+
|
|
110
|
+
samples_per_second = self.batch_size / (time.time() - last_batch_time)
|
|
111
|
+
self.log("samples_per_second", samples_per_second, epoch)
|
|
112
|
+
|
|
113
|
+
valid_loss = 0.
|
|
114
|
+
if valid_dataloader is not None:
|
|
115
|
+
# get loss on valid_dataset
|
|
116
|
+
model.eval()
|
|
117
|
+
with torch.no_grad():
|
|
118
|
+
for batch, batch_data in enumerate(valid_dataloader):
|
|
119
|
+
x, y_target = batch_data
|
|
120
|
+
x = x.float().to(self._device) # type: ignore
|
|
121
|
+
y_target = y_target.float().to(self._device)
|
|
122
|
+
y_pred, _, _ = model(x, self.lookback)
|
|
123
|
+
loss = criterion(y_pred, y_target)
|
|
124
|
+
valid_loss += loss.item()
|
|
125
|
+
|
|
126
|
+
valid_loss /= (float(batch+1))
|
|
127
|
+
model.train()
|
|
128
|
+
learning_rate_scheduler.step(valid_loss)
|
|
129
|
+
|
|
130
|
+
# save best model for later return
|
|
131
|
+
if valid_loss < min_valid_loss:
|
|
132
|
+
logger.debug(f"found best model with loss {valid_loss}")
|
|
133
|
+
best_model = model
|
|
134
|
+
best_optim = optimizer
|
|
135
|
+
min_valid_loss = valid_loss
|
|
136
|
+
self.best_loss = float(min_valid_loss)
|
|
137
|
+
|
|
138
|
+
if early_stopping(valid_loss) and self.early_stopping:
|
|
139
|
+
logger.debug("Model is not improving, Early stopping activated")
|
|
140
|
+
break
|
|
141
|
+
|
|
142
|
+
self.log("valid_loss", valid_loss, step=epoch)
|
|
143
|
+
|
|
144
|
+
lrs = []
|
|
145
|
+
for grp in optimizer.param_groups:
|
|
146
|
+
lrs.append(grp["lr"])
|
|
147
|
+
self.log("lr", lrs[0], epoch)
|
|
148
|
+
|
|
149
|
+
logger.debug(f"Epoch {epoch}/{self.epochs}: Loss/train "
|
|
150
|
+
f"{train_loss} Loss/valid {valid_loss}")
|
|
151
|
+
|
|
152
|
+
self.reset_logger()
|
|
153
|
+
return best_model, best_optim
|
|
154
|
+
|
|
155
|
+
def check_eval(self, eval_set: Any):
|
|
156
|
+
"""
|
|
157
|
+
A quick check to see how the current model evaluates on the newest
|
|
158
|
+
validation dataset
|
|
159
|
+
"""
|
|
160
|
+
valid_xs = eval_set[0][0].to_numpy()
|
|
161
|
+
valid_ys = eval_set[0][1].to_numpy()
|
|
162
|
+
valid_dataloader = self.make_dataloader(valid_xs, valid_ys, self.lookback, batch_size=1)
|
|
163
|
+
criterion = torch.nn.MSELoss()
|
|
164
|
+
valid_loss = 0.
|
|
165
|
+
# self.nn.to(self._device)
|
|
166
|
+
self.nn.eval()
|
|
167
|
+
with torch.no_grad():
|
|
168
|
+
for batch, batch_data in enumerate(valid_dataloader):
|
|
169
|
+
x, y_target = batch_data
|
|
170
|
+
x = x.float().to(self._device) # type: ignore
|
|
171
|
+
y_target = y_target.float().to(self._device)
|
|
172
|
+
y_pred, _, _ = self.nn(x, self.lookback)
|
|
173
|
+
loss = criterion(y_pred, y_target)
|
|
174
|
+
valid_loss += loss.item()
|
|
175
|
+
|
|
176
|
+
valid_loss /= (float(batch+1))
|
|
177
|
+
self.nn.train()
|
|
178
|
+
torch.cuda.empty_cache()
|
|
179
|
+
|
|
180
|
+
return valid_loss
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from typing import Any
|
|
3
|
+
import numpy.typing as npt
|
|
4
|
+
from flowml.models.pytorch.base import BasePyTorchModel
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from flowdapt.lib.logger import get_logger
|
|
7
|
+
from torch.optim.lr_scheduler import ReduceLROnPlateau
|
|
8
|
+
from flowml.models.pytorch.utils import EarlyStopping, WindowDataset
|
|
9
|
+
from flowml.models.pytorch.architectures.mlp import Net
|
|
10
|
+
|
|
11
|
+
logger = get_logger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class PyTorchMLPRegressor(BasePyTorchModel):
|
|
15
|
+
|
|
16
|
+
def fit(self,
|
|
17
|
+
xs: npt.ArrayLike | pd.DataFrame,
|
|
18
|
+
ys: npt.ArrayLike | pd.DataFrame,
|
|
19
|
+
eval_set: Any = None):
|
|
20
|
+
"""
|
|
21
|
+
High level wrapper that takes the xs, ys, and eval_set and trains
|
|
22
|
+
a model.
|
|
23
|
+
"""
|
|
24
|
+
x, y, valid_x, valid_y = self._validate_args(xs, ys, eval_set)
|
|
25
|
+
|
|
26
|
+
if x is None or y is None:
|
|
27
|
+
logger.warning("No training data provided, skipping fit")
|
|
28
|
+
|
|
29
|
+
dataset = self.make_dataloader(x, y, self.lookback)
|
|
30
|
+
if valid_x is not None:
|
|
31
|
+
valid_dataset = self.make_dataloader(valid_x, valid_y, self.lookback)
|
|
32
|
+
else:
|
|
33
|
+
valid_dataset = None
|
|
34
|
+
|
|
35
|
+
self.nn = Net(x.shape[1]*self.lookback, y.shape[1], 1).to(self._device)
|
|
36
|
+
self.nn, self.optimizer = self.train_loop(dataset, valid_dataset, self.nn)
|
|
37
|
+
|
|
38
|
+
return self
|
|
39
|
+
|
|
40
|
+
def predict(self, xs: npt.ArrayLike):
|
|
41
|
+
x, _, _, _ = self._validate_args(xs)
|
|
42
|
+
# logger.info(f"Predicting on {x.shape[0]} samples and {x.shape[1]} features")
|
|
43
|
+
|
|
44
|
+
# self.nn.to(self._device)
|
|
45
|
+
self.nn.eval()
|
|
46
|
+
data = torch.flatten(x)
|
|
47
|
+
data = data.to(self._device)
|
|
48
|
+
preds = self.nn(data)
|
|
49
|
+
preds = preds.cpu()
|
|
50
|
+
return preds.detach().numpy()
|
|
51
|
+
|
|
52
|
+
def make_dataloader(self,
|
|
53
|
+
xs: torch.Tensor,
|
|
54
|
+
ys: torch.Tensor,
|
|
55
|
+
lookback: int = 10,
|
|
56
|
+
batch_size: int | None = None):
|
|
57
|
+
"""
|
|
58
|
+
Given the xs and ys, make the dataset and dataloader for training.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
dataset = WindowDataset(xs,
|
|
62
|
+
ys,
|
|
63
|
+
window_size=lookback)
|
|
64
|
+
|
|
65
|
+
dataloader = torch.utils.data.DataLoader(
|
|
66
|
+
dataset,
|
|
67
|
+
batch_size=self.batch_size,
|
|
68
|
+
drop_last=True,
|
|
69
|
+
num_workers=0,
|
|
70
|
+
shuffle=self.shuffle
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
return dataloader
|
|
74
|
+
|
|
75
|
+
def train_loop(self, dataloader: Any, valid_dataloader: Any, model: torch.nn.Module):
|
|
76
|
+
"""
|
|
77
|
+
Provided the dataloader and valid_dataloader, train the model.
|
|
78
|
+
"""
|
|
79
|
+
model.train()
|
|
80
|
+
criterion = torch.nn.MSELoss()
|
|
81
|
+
optimizer = torch.optim.Adam(
|
|
82
|
+
model.parameters(), lr=5e-5, weight_decay=1e-4)
|
|
83
|
+
learning_rate_scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.5)
|
|
84
|
+
early_stopping = EarlyStopping(patience=50)
|
|
85
|
+
|
|
86
|
+
logger.info("Starting training loop")
|
|
87
|
+
min_valid_loss = 1e10
|
|
88
|
+
total_batches = 0
|
|
89
|
+
for epoch in range(0, self.epochs):
|
|
90
|
+
train_loss = 0.
|
|
91
|
+
for batch, batch_data in enumerate(dataloader):
|
|
92
|
+
x, y_target = batch_data
|
|
93
|
+
x = x.float().to(self._device) # type: ignore
|
|
94
|
+
y_target = y_target.float().to(self._device)
|
|
95
|
+
y_pred = model(x)
|
|
96
|
+
loss = criterion(y_pred, y_target)
|
|
97
|
+
# Backprogation
|
|
98
|
+
optimizer.zero_grad()
|
|
99
|
+
loss.backward()
|
|
100
|
+
optimizer.step()
|
|
101
|
+
total_batches += 1
|
|
102
|
+
train_loss += loss.item()
|
|
103
|
+
|
|
104
|
+
train_loss /= float(batch+1)
|
|
105
|
+
# get loss on valid_dataset
|
|
106
|
+
if valid_dataloader is not None:
|
|
107
|
+
valid_loss = 0.
|
|
108
|
+
model.eval()
|
|
109
|
+
with torch.no_grad():
|
|
110
|
+
for batch, batch_data in enumerate(valid_dataloader):
|
|
111
|
+
x, y_target = batch_data
|
|
112
|
+
x = x.float().to(self._device)
|
|
113
|
+
y_target = y_target.float().to(self._device)
|
|
114
|
+
y_pred = model(x)
|
|
115
|
+
loss = criterion(y_pred, y_target)
|
|
116
|
+
valid_loss += loss.item()
|
|
117
|
+
|
|
118
|
+
valid_loss /= float(batch+1)
|
|
119
|
+
model.train()
|
|
120
|
+
learning_rate_scheduler.step(valid_loss)
|
|
121
|
+
|
|
122
|
+
# save best model for later return
|
|
123
|
+
if valid_loss < min_valid_loss:
|
|
124
|
+
logger.debug(f"found best model with loss {valid_loss}")
|
|
125
|
+
best_model = model
|
|
126
|
+
best_optim = optimizer
|
|
127
|
+
min_valid_loss = valid_loss
|
|
128
|
+
self.best_loss = float(min_valid_loss)
|
|
129
|
+
|
|
130
|
+
if early_stopping(valid_loss) and self.early_stopping:
|
|
131
|
+
logger.debug("Model is not improving, Early stopping activated")
|
|
132
|
+
break
|
|
133
|
+
|
|
134
|
+
self.log("valid_loss", valid_loss, step=epoch)
|
|
135
|
+
|
|
136
|
+
# log epoch based metrics of interest
|
|
137
|
+
self.log("train_loss", train_loss, epoch)
|
|
138
|
+
|
|
139
|
+
lrs = []
|
|
140
|
+
for grp in optimizer.param_groups:
|
|
141
|
+
lrs.append(grp["lr"])
|
|
142
|
+
self.log("lr", lrs[0], epoch)
|
|
143
|
+
|
|
144
|
+
logger.debug(
|
|
145
|
+
f"Epoch {epoch}/{self.epochs}: Loss/train "
|
|
146
|
+
f"{train_loss} Loss/valid {valid_loss}"
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
self.reset_logger()
|
|
150
|
+
return best_model, best_optim
|
|
151
|
+
|
|
152
|
+
def check_eval(self, eval_set: Any):
|
|
153
|
+
"""
|
|
154
|
+
A quick check to see how the current model evaluates on the newest
|
|
155
|
+
validation dataset
|
|
156
|
+
"""
|
|
157
|
+
_, _, valid_xs, valid_ys = self._validate_args(eval_set=eval_set)
|
|
158
|
+
|
|
159
|
+
if valid_xs is None or valid_ys is None:
|
|
160
|
+
logger.warning("No eval_set provided, skipping eval")
|
|
161
|
+
return None
|
|
162
|
+
|
|
163
|
+
valid_dataloader = self.make_dataloader(valid_xs, valid_ys, self.lookback)
|
|
164
|
+
criterion = torch.nn.MSELoss()
|
|
165
|
+
valid_loss = 0.
|
|
166
|
+
# self.nn.to(self._device)
|
|
167
|
+
self.nn.eval()
|
|
168
|
+
with torch.no_grad():
|
|
169
|
+
for batch, batch_data in enumerate(valid_dataloader):
|
|
170
|
+
x, y_target = batch_data
|
|
171
|
+
x = x.float().to(self._device) # type: ignore
|
|
172
|
+
y_target = y_target.float().to(self._device)
|
|
173
|
+
y_pred = self.nn(x)
|
|
174
|
+
loss = criterion(y_pred, y_target)
|
|
175
|
+
valid_loss += loss.item()
|
|
176
|
+
|
|
177
|
+
valid_loss /= (float(batch+1))
|
|
178
|
+
self.nn.train()
|
|
179
|
+
torch.cuda.empty_cache()
|
|
180
|
+
return valid_loss
|