flowml 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowml/__init__.py +19 -0
- flowml/models/__init__.py +14 -0
- flowml/models/base.py +89 -0
- flowml/models/pytorch/__init__.py +9 -0
- flowml/models/pytorch/architectures/lstm.py +74 -0
- flowml/models/pytorch/architectures/mlp.py +45 -0
- flowml/models/pytorch/architectures/transformer.py +137 -0
- flowml/models/pytorch/base.py +272 -0
- flowml/models/pytorch/drivers/lstm_autoregressive.py +220 -0
- flowml/models/pytorch/drivers/lstm_direct_regressor.py +180 -0
- flowml/models/pytorch/drivers/mlp_regressor.py +180 -0
- flowml/models/pytorch/drivers/transformer_regressor.py +192 -0
- flowml/models/pytorch/utils.py +175 -0
- flowml/models/xgboost/__init__.py +7 -0
- flowml/models/xgboost/regressor.py +91 -0
- flowml/pipefill.py +160 -0
- flowml/tensorboard.py +44 -0
- flowml/utils.py +71 -0
- flowml-0.1.0.dist-info/LICENSE +201 -0
- flowml-0.1.0.dist-info/METADATA +214 -0
- flowml-0.1.0.dist-info/RECORD +22 -0
- flowml-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from typing import Any
|
|
3
|
+
import numpy.typing as npt
|
|
4
|
+
from flowml.models.pytorch.base import BasePyTorchModel
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from flowdapt.lib.logger import get_logger
|
|
7
|
+
from torch.optim.lr_scheduler import ReduceLROnPlateau
|
|
8
|
+
from flowml.models.pytorch.utils import EarlyStopping, WindowDatasetTransformer
|
|
9
|
+
from flowml.models.pytorch.architectures.transformer import TransformerPositionalEncoding
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
logger = get_logger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class PyTorchTransformer(BasePyTorchModel):
|
|
16
|
+
"""
|
|
17
|
+
A driver for a Transformer regressor.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
def fit(self, xs: npt.ArrayLike | pd.DataFrame,
|
|
21
|
+
ys: npt.ArrayLike | pd.DataFrame, eval_set: Any):
|
|
22
|
+
|
|
23
|
+
x, y, valid_x, valid_y = self._validate_args(xs, ys, eval_set)
|
|
24
|
+
|
|
25
|
+
if x is None or y is None:
|
|
26
|
+
logger.warning("No training data provided, skipping fit")
|
|
27
|
+
|
|
28
|
+
dataset = self.make_dataloader(x, y, self.lookback)
|
|
29
|
+
if valid_x is not None:
|
|
30
|
+
valid_dataset = self.make_dataloader(valid_x, valid_y, self.lookback, batch_size=1)
|
|
31
|
+
else:
|
|
32
|
+
valid_dataset = None
|
|
33
|
+
|
|
34
|
+
input_dim = x.shape[1]
|
|
35
|
+
output_dim = y.shape[1]
|
|
36
|
+
self.nn = TransformerPositionalEncoding(
|
|
37
|
+
input_dim,
|
|
38
|
+
output_dim,
|
|
39
|
+
num_layers=self.num_layers,
|
|
40
|
+
dropout=self.dropout,
|
|
41
|
+
lookback=self.lookback,
|
|
42
|
+
hidden_dim=self.hidden_dim).to(self._device)
|
|
43
|
+
# torch.compile(net)
|
|
44
|
+
self.nn, self.optimizer = self.train_loop(
|
|
45
|
+
dataset,
|
|
46
|
+
valid_dataset,
|
|
47
|
+
self.nn
|
|
48
|
+
)
|
|
49
|
+
# self.nn.to("cpu")
|
|
50
|
+
return self
|
|
51
|
+
|
|
52
|
+
def predict(self, xs: npt.ArrayLike):
|
|
53
|
+
|
|
54
|
+
x, _, _, _ = self._validate_args(xs)
|
|
55
|
+
|
|
56
|
+
self.nn.eval()
|
|
57
|
+
data = x.unsqueeze(0)
|
|
58
|
+
data = data.to(self._device)
|
|
59
|
+
preds = self.nn(data)
|
|
60
|
+
preds = preds.cpu()
|
|
61
|
+
preds = preds.squeeze().squeeze()
|
|
62
|
+
return preds.detach().numpy()
|
|
63
|
+
|
|
64
|
+
def make_dataloader(self,
|
|
65
|
+
xs: torch.Tensor,
|
|
66
|
+
ys: torch.Tensor,
|
|
67
|
+
lookback: int = 10,
|
|
68
|
+
batch_size: int | None = None):
|
|
69
|
+
|
|
70
|
+
if batch_size is None:
|
|
71
|
+
batch_size = self.batch_size
|
|
72
|
+
|
|
73
|
+
dataset = WindowDatasetTransformer(
|
|
74
|
+
xs,
|
|
75
|
+
ys,
|
|
76
|
+
window_size=lookback
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
dataloader = torch.utils.data.DataLoader(
|
|
80
|
+
dataset,
|
|
81
|
+
batch_size=batch_size,
|
|
82
|
+
drop_last=True,
|
|
83
|
+
num_workers=0,
|
|
84
|
+
shuffle=self.shuffle
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
return dataloader
|
|
88
|
+
|
|
89
|
+
def train_loop(self, dataloader: Any, valid_dataloader: Any, model: torch.nn.Module):
|
|
90
|
+
|
|
91
|
+
model.train()
|
|
92
|
+
criterion = torch.nn.MSELoss()
|
|
93
|
+
optimizer = torch.optim.Adam(
|
|
94
|
+
model.parameters(), lr=1e-4, weight_decay=1e-4)
|
|
95
|
+
learning_rate_scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.5, patience=5)
|
|
96
|
+
early_stopping = EarlyStopping(patience=20)
|
|
97
|
+
|
|
98
|
+
logger.info("Starting training loop")
|
|
99
|
+
min_valid_loss = 1e10
|
|
100
|
+
total_batches = 0
|
|
101
|
+
for epoch in range(0, self.epochs):
|
|
102
|
+
train_loss = 0.
|
|
103
|
+
last_batch_time = time.time()
|
|
104
|
+
for batch, batch_data in enumerate(dataloader):
|
|
105
|
+
x, y_target = batch_data
|
|
106
|
+
x = x.float().to(self._device) # type: ignore
|
|
107
|
+
y_target = y_target.float().to(self._device)
|
|
108
|
+
y_pred = model(x)
|
|
109
|
+
loss = criterion(y_pred, y_target)
|
|
110
|
+
# Backprogation
|
|
111
|
+
optimizer.zero_grad()
|
|
112
|
+
loss.backward()
|
|
113
|
+
optimizer.step()
|
|
114
|
+
total_batches += 1
|
|
115
|
+
train_loss += loss.item()
|
|
116
|
+
self.log("train_loss", loss.item(), total_batches)
|
|
117
|
+
total_batches += 1
|
|
118
|
+
|
|
119
|
+
samples_per_second = self.batch_size / (time.time() - last_batch_time)
|
|
120
|
+
self.log("samples_per_second", samples_per_second, epoch)
|
|
121
|
+
|
|
122
|
+
if valid_dataloader is not None:
|
|
123
|
+
# get loss on valid_dataset
|
|
124
|
+
valid_loss = 0.
|
|
125
|
+
model.eval()
|
|
126
|
+
with torch.no_grad():
|
|
127
|
+
for batch, batch_data in enumerate(valid_dataloader):
|
|
128
|
+
x, y_target = batch_data
|
|
129
|
+
x = x.float().to(self._device) # type: ignore
|
|
130
|
+
y_target = y_target.float().to(self._device)
|
|
131
|
+
y_pred = model(x)
|
|
132
|
+
loss = criterion(y_pred, y_target)
|
|
133
|
+
valid_loss += loss.item()
|
|
134
|
+
|
|
135
|
+
valid_loss /= (float(batch+1))
|
|
136
|
+
model.train()
|
|
137
|
+
learning_rate_scheduler.step(valid_loss)
|
|
138
|
+
|
|
139
|
+
# save best model for later return
|
|
140
|
+
if valid_loss < min_valid_loss:
|
|
141
|
+
logger.debug(f"found best model with loss {valid_loss}")
|
|
142
|
+
best_model = model
|
|
143
|
+
best_optim = optimizer
|
|
144
|
+
min_valid_loss = valid_loss
|
|
145
|
+
self.best_loss = float(min_valid_loss)
|
|
146
|
+
|
|
147
|
+
if early_stopping(valid_loss) and self.early_stopping:
|
|
148
|
+
logger.debug("Model is not improving, Early stopping activated")
|
|
149
|
+
break
|
|
150
|
+
|
|
151
|
+
self.log("valid_loss", valid_loss, step=epoch)
|
|
152
|
+
|
|
153
|
+
lrs = []
|
|
154
|
+
for grp in optimizer.param_groups:
|
|
155
|
+
lrs.append(grp["lr"])
|
|
156
|
+
self.log("lr", lrs[0], epoch)
|
|
157
|
+
|
|
158
|
+
logger.debug(f"Epoch {epoch}/{self.epochs}: Loss/train "
|
|
159
|
+
f"{train_loss} Loss/valid {valid_loss}")
|
|
160
|
+
|
|
161
|
+
self.reset_logger()
|
|
162
|
+
return best_model, best_optim
|
|
163
|
+
|
|
164
|
+
def check_eval(self, eval_set: Any):
|
|
165
|
+
"""
|
|
166
|
+
A quick check to see how the current model evaluates on the newest
|
|
167
|
+
validation dataset
|
|
168
|
+
"""
|
|
169
|
+
_, _, valid_xs, valid_ys = self._validate_args(eval_set=eval_set)
|
|
170
|
+
|
|
171
|
+
if valid_xs is None or valid_ys is None:
|
|
172
|
+
logger.warning("No eval_set provided, skipping eval")
|
|
173
|
+
return None
|
|
174
|
+
|
|
175
|
+
valid_dataloader = self.make_dataloader(valid_xs, valid_ys, self.lookback, batch_size=1)
|
|
176
|
+
criterion = torch.nn.MSELoss()
|
|
177
|
+
valid_loss = 0.
|
|
178
|
+
# self.nn.to(self._device)
|
|
179
|
+
self.nn.eval()
|
|
180
|
+
with torch.no_grad():
|
|
181
|
+
for batch, batch_data in enumerate(valid_dataloader):
|
|
182
|
+
x, y_target = batch_data
|
|
183
|
+
x = x.float().to(self._device) # type: ignore
|
|
184
|
+
y_target = y_target.float().to(self._device)
|
|
185
|
+
y_pred = self.nn(x)
|
|
186
|
+
loss = criterion(y_pred, y_target)
|
|
187
|
+
valid_loss += loss.item()
|
|
188
|
+
|
|
189
|
+
valid_loss /= (float(batch+1))
|
|
190
|
+
self.nn.train()
|
|
191
|
+
# torch.cuda.empty_cache()
|
|
192
|
+
return valid_loss
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import multiprocessing
|
|
2
|
+
import torch
|
|
3
|
+
from flowdapt.lib.logger import get_logger
|
|
4
|
+
import math
|
|
5
|
+
|
|
6
|
+
logger = get_logger(__name__)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class PositionalEncoding(torch.nn.Module):
|
|
10
|
+
def __init__(self, d_model, max_len=5000):
|
|
11
|
+
"""
|
|
12
|
+
Args
|
|
13
|
+
d_model: Hidden dimensionality of the input.
|
|
14
|
+
max_len: Maximum length of a sequence to expect.
|
|
15
|
+
"""
|
|
16
|
+
super().__init__()
|
|
17
|
+
|
|
18
|
+
# Create matrix of [SeqLen, HiddenDim] representing the
|
|
19
|
+
# positional encoding for max_len inputs
|
|
20
|
+
pe = torch.zeros(max_len, d_model)
|
|
21
|
+
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
|
|
22
|
+
div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
|
|
23
|
+
pe[:, 0::2] = torch.sin(position * div_term)
|
|
24
|
+
pe[:, 1::2] = torch.cos(position * div_term)
|
|
25
|
+
pe = pe.unsqueeze(0)
|
|
26
|
+
|
|
27
|
+
self.register_buffer("pe", pe, persistent=False)
|
|
28
|
+
|
|
29
|
+
def forward(self, x):
|
|
30
|
+
x = x + self.pe[:, : x.size(1)]
|
|
31
|
+
return x
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class WindowDataset(torch.utils.data.Dataset):
|
|
35
|
+
"""
|
|
36
|
+
Given the lookback (window_size), traverses the data
|
|
37
|
+
in reverse order and returns a the X and y on __getitem__
|
|
38
|
+
"""
|
|
39
|
+
def __init__(self, xs, ys, window_size):
|
|
40
|
+
self.xs = xs
|
|
41
|
+
self.ys = ys
|
|
42
|
+
self.window_size = window_size
|
|
43
|
+
|
|
44
|
+
def __len__(self):
|
|
45
|
+
return len(self.xs) - self.window_size
|
|
46
|
+
|
|
47
|
+
def __getitem__(self, index):
|
|
48
|
+
idx_rev = len(self.xs) - self.window_size - index - 1
|
|
49
|
+
window_x = torch.flatten(self.xs[idx_rev:idx_rev+self.window_size, :])
|
|
50
|
+
# Beware of indexing, these two window_x and window_y are aimed at the same row!
|
|
51
|
+
# this is what happens when you use ":"
|
|
52
|
+
window_y = self.ys[idx_rev+self.window_size-1, :]
|
|
53
|
+
return window_x, window_y
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class WindowDatasetTransformer(torch.utils.data.Dataset):
|
|
57
|
+
def __init__(self, xs, ys, window_size):
|
|
58
|
+
self.xs = xs
|
|
59
|
+
self.ys = ys
|
|
60
|
+
self.window_size = window_size
|
|
61
|
+
|
|
62
|
+
def __len__(self):
|
|
63
|
+
return len(self.xs) - self.window_size
|
|
64
|
+
|
|
65
|
+
def __getitem__(self, index):
|
|
66
|
+
idx_rev = len(self.xs) - self.window_size - index - 1
|
|
67
|
+
window_x = self.xs[idx_rev:idx_rev+self.window_size, :]
|
|
68
|
+
# Beware of indexing, these two window_x and window_y are aimed at the same row!
|
|
69
|
+
# this is what happens when you use :
|
|
70
|
+
window_y = self.ys[idx_rev+self.window_size-1, :].unsqueeze(0)
|
|
71
|
+
return window_x, window_y
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class WindowDatasetTransformerTraditional(torch.utils.data.Dataset):
|
|
75
|
+
def __init__(self, xs, ys, window_size, rollout=6):
|
|
76
|
+
self.xs = xs
|
|
77
|
+
self.ys = ys
|
|
78
|
+
self.window_size = window_size
|
|
79
|
+
self.rollout = rollout
|
|
80
|
+
|
|
81
|
+
def __len__(self):
|
|
82
|
+
return len(self.xs) - self.window_size - self.rollout - 1
|
|
83
|
+
|
|
84
|
+
def __getitem__(self, index):
|
|
85
|
+
idx_rev = len(self.xs) - self.window_size - index - 1
|
|
86
|
+
window_x = self.xs[idx_rev:idx_rev+self.window_size, :]
|
|
87
|
+
# Beware of indexing, these two window_x and window_y are aimed at the same row!
|
|
88
|
+
# this is what happens when you use :
|
|
89
|
+
window_y = self.ys[idx_rev+self.window_size:self.rollout, :]
|
|
90
|
+
return window_x, window_y
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class WindowAutoRegressiveDataset(torch.utils.data.Dataset):
|
|
94
|
+
def __init__(self, xs, window_size, rollout, valid: bool = False):
|
|
95
|
+
self.xs = xs
|
|
96
|
+
self.window_size = window_size
|
|
97
|
+
self.rollout = rollout
|
|
98
|
+
self.validation = valid
|
|
99
|
+
|
|
100
|
+
def __len__(self):
|
|
101
|
+
if not self.validation:
|
|
102
|
+
return len(self.xs) - self.window_size - self.rollout - 1
|
|
103
|
+
# number of samples for training is the total points divided by the
|
|
104
|
+
# full rollout for a single rollout
|
|
105
|
+
else:
|
|
106
|
+
return len(self.xs) - self.window_size - self.rollout - 1
|
|
107
|
+
# return int(math.floor(len(self.xs) / (self.window_size + self.rollout)))
|
|
108
|
+
# else:
|
|
109
|
+
# # number of samples is as many as possible for validation
|
|
110
|
+
# return len(self.xs) - self.window_size - self.rollout - 1
|
|
111
|
+
|
|
112
|
+
def __getitem__(self, index):
|
|
113
|
+
|
|
114
|
+
if self.validation:
|
|
115
|
+
idx = index # * (self.window_size + self.rollout)
|
|
116
|
+
else:
|
|
117
|
+
idx = index
|
|
118
|
+
|
|
119
|
+
# idx_rev = len(self.xs) - self.window_size - self.rollout - index - 1
|
|
120
|
+
window_x = self.xs[idx:idx+self.window_size, :]
|
|
121
|
+
# Beware of indexing, these two window_x and window_y are aimed at subsequent rows!
|
|
122
|
+
# this is what happens when you use : for indexing
|
|
123
|
+
y_start = idx + self.window_size
|
|
124
|
+
y_end = y_start + self.rollout
|
|
125
|
+
window_y = self.xs[y_start:y_end, :]
|
|
126
|
+
return window_x, window_y
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class EarlyStopping:
|
|
130
|
+
"""Stops early if the validation loss has plateaued"""
|
|
131
|
+
|
|
132
|
+
def __init__(self, patience=10, delta=0):
|
|
133
|
+
"""
|
|
134
|
+
Args:
|
|
135
|
+
patience (int): How long to wait after last time validation loss improved.
|
|
136
|
+
Default: 10
|
|
137
|
+
delta (float): Minimum change in the monitored quantity to qualify as an improvement.
|
|
138
|
+
Default: 0
|
|
139
|
+
"""
|
|
140
|
+
self.patience = patience
|
|
141
|
+
self.counter = 0
|
|
142
|
+
self.best_score = None
|
|
143
|
+
self.early_stop = False
|
|
144
|
+
self.delta = delta
|
|
145
|
+
|
|
146
|
+
def __call__(self, val_loss):
|
|
147
|
+
|
|
148
|
+
score = -val_loss
|
|
149
|
+
|
|
150
|
+
if self.best_score is None:
|
|
151
|
+
self.best_score = score
|
|
152
|
+
elif score < self.best_score + self.delta:
|
|
153
|
+
self.counter += 1
|
|
154
|
+
logger.debug(f"EarlyStopping counter: {self.counter} out of {self.patience}")
|
|
155
|
+
if self.counter >= self.patience:
|
|
156
|
+
self.early_stop = True
|
|
157
|
+
else:
|
|
158
|
+
self.best_score = score
|
|
159
|
+
self.counter = 0
|
|
160
|
+
|
|
161
|
+
return self.early_stop
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def check_for_gpu():
|
|
165
|
+
if torch.cuda.is_available() and torch.cuda.device_count() > 0:
|
|
166
|
+
device = torch.device("cuda")
|
|
167
|
+
logger.debug("Found cuda enabled gpu")
|
|
168
|
+
else:
|
|
169
|
+
device = torch.device("cpu")
|
|
170
|
+
|
|
171
|
+
return device
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def set_num_threads(num_threads):
|
|
175
|
+
torch.set_num_threads(num_threads)
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from typing import Any, Callable
|
|
2
|
+
from joblib import dump, load
|
|
3
|
+
from xgboost.dask import DaskXGBRegressor
|
|
4
|
+
from xgboost import XGBRegressor
|
|
5
|
+
|
|
6
|
+
from flowml.models.base import BaseMLModel
|
|
7
|
+
from flowdapt.compute.artifacts import Artifact
|
|
8
|
+
from flowml.tensorboard import TensorBoardCallback
|
|
9
|
+
from flowdapt.lib.logger import get_logger
|
|
10
|
+
from sklearn.metrics import mean_squared_error
|
|
11
|
+
import os
|
|
12
|
+
import tempfile
|
|
13
|
+
|
|
14
|
+
import warnings
|
|
15
|
+
|
|
16
|
+
# Suppress XGBoost warnings
|
|
17
|
+
warnings.filterwarnings(action='ignore', module='xgboost')
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
logger = get_logger(__name__)
|
|
21
|
+
|
|
22
|
+
class XGBoostRegressor(BaseMLModel):
|
|
23
|
+
def __init_model__(self, **kwargs) -> "XGBoostRegressor":
|
|
24
|
+
self._model = XGBRegressor(**kwargs)
|
|
25
|
+
return self
|
|
26
|
+
|
|
27
|
+
def get_params(self) -> dict:
|
|
28
|
+
return self._model.get_params()
|
|
29
|
+
|
|
30
|
+
@classmethod
|
|
31
|
+
def from_artifact(cls, **kwargs) -> Callable[[Artifact], Any]:
|
|
32
|
+
def _inner(artifact: Artifact):
|
|
33
|
+
# Require model artifact
|
|
34
|
+
if not artifact.get_meta("type", None) == "model":
|
|
35
|
+
raise ValueError("Artifact must be of type `model`")
|
|
36
|
+
|
|
37
|
+
# Get the meta from the Artifact
|
|
38
|
+
model_params = artifact.get_meta("params", {})
|
|
39
|
+
kwargs.update(model_params)
|
|
40
|
+
|
|
41
|
+
file = artifact.get_file("model.joblib")
|
|
42
|
+
|
|
43
|
+
with file.open(mode="rb") as f:
|
|
44
|
+
model = load(f)
|
|
45
|
+
|
|
46
|
+
# Create the instance and assign the model
|
|
47
|
+
instance = cls(name=artifact.name, **model_params)
|
|
48
|
+
instance._model = model
|
|
49
|
+
|
|
50
|
+
return instance
|
|
51
|
+
return _inner
|
|
52
|
+
|
|
53
|
+
def to_artifact(self, **kwargs) -> Callable[[Artifact, Any], None]:
|
|
54
|
+
def _inner(artifact: Artifact, value: Any):
|
|
55
|
+
# Add the meta to the artifact
|
|
56
|
+
artifact["type"] = "model"
|
|
57
|
+
artifact["params"] = self.get_params()
|
|
58
|
+
artifact["model_type"] = self.get_type()
|
|
59
|
+
|
|
60
|
+
with artifact.new_file("model.joblib", exist_ok=True).open(mode="wb") as f:
|
|
61
|
+
dump(self._model, f)
|
|
62
|
+
return _inner
|
|
63
|
+
|
|
64
|
+
def fit(self, *args, **kwargs):
|
|
65
|
+
eval_set = kwargs["eval_set"]
|
|
66
|
+
if self.default_logger:
|
|
67
|
+
self.current_log_dir = os.path.join(tempfile.gettempdir(), self.name_str)
|
|
68
|
+
os.makedirs(self.current_log_dir, exist_ok=True)
|
|
69
|
+
self._model.set_params(callbacks=[
|
|
70
|
+
TensorBoardCallback(self.current_log_dir)
|
|
71
|
+
])
|
|
72
|
+
self._model.fit(*args, verbose=False, **kwargs)
|
|
73
|
+
if self.default_logger:
|
|
74
|
+
self.logger_to_artifact()
|
|
75
|
+
self._model.set_params(callbacks=[])
|
|
76
|
+
self.best_loss = self.check_eval(eval_set)
|
|
77
|
+
return self
|
|
78
|
+
|
|
79
|
+
def predict(self, *args, **kwargs):
|
|
80
|
+
return self._model.predict(*args, **kwargs)
|
|
81
|
+
|
|
82
|
+
def check_eval(self, eval_set: Any):
|
|
83
|
+
ys = self._model.predict(eval_set[0][0])
|
|
84
|
+
valid_loss = mean_squared_error(eval_set[0][1], ys)
|
|
85
|
+
logger.info(f"Validation loss: {valid_loss}")
|
|
86
|
+
return valid_loss
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class DaskXGBoostRegressor(XGBoostRegressor):
|
|
90
|
+
def __init_model__(self, name_str, **kwargs):
|
|
91
|
+
return DaskXGBRegressor(**kwargs)
|
flowml/pipefill.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
import cloudpickle
|
|
3
|
+
import numpy.typing as npt
|
|
4
|
+
import numpy as np
|
|
5
|
+
from flowdapt.lib.logger import get_logger
|
|
6
|
+
from typing import Type, Any
|
|
7
|
+
from datasieve.pipeline import Pipeline
|
|
8
|
+
from datetime import datetime, timezone
|
|
9
|
+
|
|
10
|
+
from flowdapt.compute.artifacts import Artifact
|
|
11
|
+
from flowml.models.base import BaseMLModel
|
|
12
|
+
|
|
13
|
+
from flowdapt.lib.utils.misc import import_from_string
|
|
14
|
+
from flowdapt.compute.artifacts import get_artifact
|
|
15
|
+
# from io import BytesIO
|
|
16
|
+
# from flowml.models.pytorch.utils import check_for_gpu
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
logger = get_logger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class PipeFill:
|
|
23
|
+
"""
|
|
24
|
+
Class designed to hold all necessary pipeline components to
|
|
25
|
+
more easily move and organize objects at large-scale thousand
|
|
26
|
+
model+ environments.
|
|
27
|
+
It wraps a flowml BaseMLModel object to make cluster-memory
|
|
28
|
+
and distributed storage easier and more efficient.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
name_str: str = "",
|
|
34
|
+
namespace: str = "",
|
|
35
|
+
model_str: str = "flowml.XGBoostRegressor",
|
|
36
|
+
model_train_params: dict[str, Any] = {},
|
|
37
|
+
data_split_params: dict[str, Any] = {},
|
|
38
|
+
extras: dict[str, Any] = {}
|
|
39
|
+
):
|
|
40
|
+
|
|
41
|
+
self.model_train_params = model_train_params
|
|
42
|
+
self.data_split_parameters = data_split_params
|
|
43
|
+
self.extras = extras
|
|
44
|
+
self.last_trained_timestamp: int = 0
|
|
45
|
+
self.model_str: str = model_str
|
|
46
|
+
# FIXME: change name from MODELCLASS to something better
|
|
47
|
+
self.MODELCLASS: Type[BaseMLModel] = import_from_string(model_str)
|
|
48
|
+
self.model: BaseMLModel = self.MODELCLASS(
|
|
49
|
+
name_str=name_str,
|
|
50
|
+
namespace=namespace,
|
|
51
|
+
**model_train_params
|
|
52
|
+
)
|
|
53
|
+
self.namespace: str = namespace
|
|
54
|
+
self.model_artifact: Artifact | None = None # ignore: type
|
|
55
|
+
self.id: str | None = name_str
|
|
56
|
+
self.do_predict: npt.ArrayLike = np.array([])
|
|
57
|
+
self.feature_pipeline: Pipeline = Pipeline()
|
|
58
|
+
self.target_pipeline: Pipeline = Pipeline()
|
|
59
|
+
self.metadata: dict[str, Any] = {}
|
|
60
|
+
self.feature_list: list[str] = []
|
|
61
|
+
self.label_list: list[str] = []
|
|
62
|
+
self.best_loss: float = 0
|
|
63
|
+
self.needs_training: bool = False
|
|
64
|
+
self.update_count: int = 0
|
|
65
|
+
self.update_count_list: list = []
|
|
66
|
+
|
|
67
|
+
def set_trained_timestamp(self):
|
|
68
|
+
self.trained_timestamp = datetime.now(tz=timezone.utc).timestamp()
|
|
69
|
+
|
|
70
|
+
def clear_data(self):
|
|
71
|
+
"""
|
|
72
|
+
Avoid saving too much data. We need to be careful here,
|
|
73
|
+
some outlier detection methods may require some of this
|
|
74
|
+
data. Therefore, we need to recreate the data, or load
|
|
75
|
+
it from disk using the database.
|
|
76
|
+
"""
|
|
77
|
+
self.model = None
|
|
78
|
+
|
|
79
|
+
def to_dict(self):
|
|
80
|
+
class_dict = vars(self).copy()
|
|
81
|
+
class_dict.pop("MODELCLASS")
|
|
82
|
+
class_dict.pop("model", None)
|
|
83
|
+
return class_dict
|
|
84
|
+
|
|
85
|
+
# def __getstate__(self):
|
|
86
|
+
# buffer = BytesIO()
|
|
87
|
+
# if self.model:
|
|
88
|
+
# model_state = self.model.__getstate__()
|
|
89
|
+
# else:
|
|
90
|
+
# model_state = None
|
|
91
|
+
# self.clear_data()
|
|
92
|
+
# state = self.to_dict()
|
|
93
|
+
# state["model"] = model_state
|
|
94
|
+
|
|
95
|
+
# buffer.write(cloudpickle.dumps(state))
|
|
96
|
+
# buffer.seek(0)
|
|
97
|
+
# return buffer.getvalue()
|
|
98
|
+
|
|
99
|
+
# def __setstate__(self, state):
|
|
100
|
+
# buffer = BytesIO(state)
|
|
101
|
+
# state = cloudpickle.load(buffer)
|
|
102
|
+
|
|
103
|
+
# for key, value in state.items():
|
|
104
|
+
# setattr(self, key, value)
|
|
105
|
+
|
|
106
|
+
# self.MODELCLASS = import_from_string(self.model_str)
|
|
107
|
+
|
|
108
|
+
# self.model = self.MODELCLASS()
|
|
109
|
+
# if state["model"] is not None:
|
|
110
|
+
# self.model.__setstate__(state["model"])
|
|
111
|
+
|
|
112
|
+
@classmethod
|
|
113
|
+
def from_artifact(cls, **kwargs):
|
|
114
|
+
"""
|
|
115
|
+
Load a PipeFill from an Artifact.
|
|
116
|
+
"""
|
|
117
|
+
# We can make use of any kind of kwargs passed at load time
|
|
118
|
+
# from object_store.get(load_artifact_hook=PipeFill.from_artifact(some_kwarg=some_value))
|
|
119
|
+
# but _inner won't actually be called until the artifact is loaded
|
|
120
|
+
def _inner(artifact: Artifact):
|
|
121
|
+
if not artifact.get_meta("type") == "pipefill":
|
|
122
|
+
raise ValueError("Artifact must be of type `pipefill`")
|
|
123
|
+
|
|
124
|
+
instance_pickle = artifact.get_file("pipefill.pkl")
|
|
125
|
+
|
|
126
|
+
# Deserialize the pickled model
|
|
127
|
+
instance = cloudpickle.loads(instance_pickle.read())
|
|
128
|
+
model_artifact = get_artifact(
|
|
129
|
+
f"{artifact.name}_model",
|
|
130
|
+
namespace=artifact._namespace
|
|
131
|
+
)
|
|
132
|
+
instance.model = instance.MODELCLASS.from_artifact()(model_artifact)
|
|
133
|
+
return instance
|
|
134
|
+
return _inner
|
|
135
|
+
|
|
136
|
+
def to_artifact(self, **kwargs):
|
|
137
|
+
"""
|
|
138
|
+
Save a PipeFill to an Artifact.
|
|
139
|
+
"""
|
|
140
|
+
# We can make use of any kind of kwargs passed at load time
|
|
141
|
+
# from object_store.get(load_artifact_hook=PipeFill.from_artifact(some_kwarg=some_value))
|
|
142
|
+
# but _inner won't actually be called until the artifact is loaded
|
|
143
|
+
def _inner(artifact: Artifact, _: Any):
|
|
144
|
+
artifact["type"] = "pipefill"
|
|
145
|
+
model_artifact = get_artifact(
|
|
146
|
+
f"{artifact.name}_model",
|
|
147
|
+
namespace=self.namespace,
|
|
148
|
+
create=True
|
|
149
|
+
)
|
|
150
|
+
self.model.to_artifact()(model_artifact, None)
|
|
151
|
+
# we save the model separately above, so now we clear it
|
|
152
|
+
self.clear_data()
|
|
153
|
+
|
|
154
|
+
# Create the serialized model object
|
|
155
|
+
serialized_instance = cloudpickle.dumps(self)
|
|
156
|
+
|
|
157
|
+
# Save the serialized model to the `model.pkl` file
|
|
158
|
+
pkl = artifact.new_file("pipefill.pkl")
|
|
159
|
+
pkl.write(serialized_instance)
|
|
160
|
+
return _inner
|
flowml/tensorboard.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from torch.utils.tensorboard import SummaryWriter
|
|
2
|
+
from typing import Any
|
|
3
|
+
import xgboost as xgb
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TensorboardLogger:
|
|
7
|
+
def __init__(self, logdir: str = "tensorboard", id: str = "unique-id"):
|
|
8
|
+
self.writer: SummaryWriter = SummaryWriter(f"{logdir}/model_{id}")
|
|
9
|
+
|
|
10
|
+
def add_scalar(self, tag: str, scalar_value: Any, step: int):
|
|
11
|
+
self.writer.add_scalar(tag, scalar_value, step)
|
|
12
|
+
|
|
13
|
+
def close(self):
|
|
14
|
+
self.writer.flush()
|
|
15
|
+
self.writer.close()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TensorBoardCallback(xgb.callback.TrainingCallback):
|
|
19
|
+
|
|
20
|
+
def __init__(self, logdir: str = "tensorboard", id: str = "unique-id"):
|
|
21
|
+
self.writer: SummaryWriter = SummaryWriter(logdir)
|
|
22
|
+
|
|
23
|
+
def after_iteration(
|
|
24
|
+
self, model, epoch: int, evals_log: xgb.callback.TrainingCallback.EvalsLog
|
|
25
|
+
) -> bool:
|
|
26
|
+
if not evals_log:
|
|
27
|
+
return False
|
|
28
|
+
|
|
29
|
+
for data, metric in evals_log.items():
|
|
30
|
+
for metric_name, log in metric.items():
|
|
31
|
+
score = log[-1][0] if isinstance(log[-1], tuple) else log[-1]
|
|
32
|
+
if data == "train":
|
|
33
|
+
# FIXME: ensure we can get the MSE for XGBoost without squaring this
|
|
34
|
+
self.writer.add_scalar("train_loss", score, epoch)
|
|
35
|
+
else:
|
|
36
|
+
self.writer.add_scalar("valid_loss", score, epoch)
|
|
37
|
+
|
|
38
|
+
return False
|
|
39
|
+
|
|
40
|
+
def after_training(self, model):
|
|
41
|
+
self.writer.flush()
|
|
42
|
+
self.writer.close()
|
|
43
|
+
|
|
44
|
+
return model
|