flowml 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowml/__init__.py +19 -0
- flowml/models/__init__.py +14 -0
- flowml/models/base.py +89 -0
- flowml/models/pytorch/__init__.py +9 -0
- flowml/models/pytorch/architectures/lstm.py +74 -0
- flowml/models/pytorch/architectures/mlp.py +45 -0
- flowml/models/pytorch/architectures/transformer.py +137 -0
- flowml/models/pytorch/base.py +272 -0
- flowml/models/pytorch/drivers/lstm_autoregressive.py +220 -0
- flowml/models/pytorch/drivers/lstm_direct_regressor.py +180 -0
- flowml/models/pytorch/drivers/mlp_regressor.py +180 -0
- flowml/models/pytorch/drivers/transformer_regressor.py +192 -0
- flowml/models/pytorch/utils.py +175 -0
- flowml/models/xgboost/__init__.py +7 -0
- flowml/models/xgboost/regressor.py +91 -0
- flowml/pipefill.py +160 -0
- flowml/tensorboard.py +44 -0
- flowml/utils.py +71 -0
- flowml-0.1.0.dist-info/LICENSE +201 -0
- flowml-0.1.0.dist-info/METADATA +214 -0
- flowml-0.1.0.dist-info/RECORD +22 -0
- flowml-0.1.0.dist-info/WHEEL +4 -0
flowml/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from flowml.models.pytorch.drivers.lstm_direct_regressor import PyTorchLSTMDirect
|
|
2
|
+
from flowml.models.pytorch.drivers.lstm_autoregressive import PyTorchLSTMAutoregressive
|
|
3
|
+
from flowml.models.pytorch.drivers.mlp_regressor import PyTorchMLPRegressor
|
|
4
|
+
from flowml.models.pytorch.drivers.transformer_regressor import PyTorchTransformer
|
|
5
|
+
from flowml.models.xgboost.regressor import XGBoostRegressor
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
__all__ = (
|
|
9
|
+
"PyTorchMLPRegressor",
|
|
10
|
+
"PyTorchTransformer",
|
|
11
|
+
"PyTorchLSTMAutoregressive",
|
|
12
|
+
"PyTorchLSTMDirect",
|
|
13
|
+
"XGBoostRegressor",
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
# Get version from pyproject.toml
|
|
17
|
+
from importlib import metadata # noqa
|
|
18
|
+
__version__ = metadata.version(__package__)
|
|
19
|
+
del metadata
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from flowml.models.pytorch.drivers.lstm_direct_regressor import PyTorchLSTMDirect
|
|
2
|
+
from flowml.models.pytorch.drivers.lstm_autoregressive import PyTorchLSTMAutoregressive
|
|
3
|
+
from flowml.models.pytorch.drivers.mlp_regressor import PyTorchMLPRegressor
|
|
4
|
+
from flowml.models.pytorch.drivers.transformer_regressor import PyTorchTransformer
|
|
5
|
+
from flowml.models.xgboost.regressor import XGBoostRegressor
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
__all__ = (
|
|
9
|
+
"PyTorchMLPRegressor",
|
|
10
|
+
"PyTorchTransformer",
|
|
11
|
+
"PyTorchLSTMAutoregressive",
|
|
12
|
+
"PyTorchLSTMDirect",
|
|
13
|
+
"XGBoostRegressor",
|
|
14
|
+
)
|
flowml/models/base.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import Any, Callable
|
|
3
|
+
|
|
4
|
+
from flowdapt.compute.artifacts import Artifact
|
|
5
|
+
from flowdapt.lib.utils.misc import generate_name
|
|
6
|
+
from flowdapt.compute.artifacts import get_artifact
|
|
7
|
+
import glob
|
|
8
|
+
import shutil
|
|
9
|
+
from flowdapt.lib.logger import get_logger
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
|
|
12
|
+
logger = get_logger(__name__)
|
|
13
|
+
|
|
14
|
+
class BaseMLModel(ABC):
|
|
15
|
+
"""
|
|
16
|
+
The base interface a Model subclass must define.
|
|
17
|
+
"""
|
|
18
|
+
name: str
|
|
19
|
+
meta: dict[str, Any] = {}
|
|
20
|
+
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
name_str: str | None = None,
|
|
24
|
+
namespace: str | None = None,
|
|
25
|
+
default_logger: bool = False,
|
|
26
|
+
**kwargs
|
|
27
|
+
) -> None:
|
|
28
|
+
self.default_logger = default_logger
|
|
29
|
+
self.__init_model__(**kwargs)
|
|
30
|
+
self._namespace = namespace
|
|
31
|
+
self._model_train_params = kwargs
|
|
32
|
+
self.name_str = name_str or generate_name()
|
|
33
|
+
self.meta = {
|
|
34
|
+
"params": self.get_params(),
|
|
35
|
+
"namespace": self._namespace,
|
|
36
|
+
}
|
|
37
|
+
self.current_log_dir: str | None = None
|
|
38
|
+
|
|
39
|
+
def get_type(self):
|
|
40
|
+
return f"{self.__module__}.{self.__class__.__name__}"
|
|
41
|
+
|
|
42
|
+
@abstractmethod
|
|
43
|
+
def fit(self, *args, **kwargs):
|
|
44
|
+
...
|
|
45
|
+
|
|
46
|
+
@abstractmethod
|
|
47
|
+
def predict(self, *args, **kwargs):
|
|
48
|
+
...
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
@abstractmethod
|
|
52
|
+
def from_artifact(cls, **kwargs) -> Callable[[Artifact], Any]:
|
|
53
|
+
...
|
|
54
|
+
|
|
55
|
+
@abstractmethod
|
|
56
|
+
def to_artifact(self, **kwargs) -> Callable[[Artifact, Any], None]:
|
|
57
|
+
...
|
|
58
|
+
|
|
59
|
+
@abstractmethod
|
|
60
|
+
def __init_model__(self, *args, **kwargs):
|
|
61
|
+
...
|
|
62
|
+
|
|
63
|
+
@abstractmethod
|
|
64
|
+
def get_params(self):
|
|
65
|
+
...
|
|
66
|
+
|
|
67
|
+
def logger_to_artifact(self, **kwargs):
|
|
68
|
+
"""
|
|
69
|
+
Copy the local temp tensorboard log to artifact for safe keeping
|
|
70
|
+
Create a unique name for the artifact so that it is easily
|
|
71
|
+
managed by tensorboard
|
|
72
|
+
"""
|
|
73
|
+
class_str = self.__class__.__name__
|
|
74
|
+
time_str = int(datetime.now().timestamp())
|
|
75
|
+
artifact_name = (f"{class_str}_{self.name_str}_{time_str}")
|
|
76
|
+
|
|
77
|
+
logger.warning("Saving tensorboard logs to artifact")
|
|
78
|
+
events_files = glob.glob(f"{self.current_log_dir}/*.tfevents*")
|
|
79
|
+
if events_files:
|
|
80
|
+
events_file = events_files[0]
|
|
81
|
+
events_file_name = events_file.split("/")[-1]
|
|
82
|
+
|
|
83
|
+
artifact = get_artifact(artifact_name, create=True)
|
|
84
|
+
with open(events_file, "rb") as local_file:
|
|
85
|
+
with artifact.new_file(events_file_name).open("wb") as artifact_file:
|
|
86
|
+
artifact_file.write(local_file.read())
|
|
87
|
+
|
|
88
|
+
# remove the local directory self.current_log_dir
|
|
89
|
+
shutil.rmtree(self.current_log_dir)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# from flowml.models.pytorch.drivers.regressor import PyTorchRegressor
|
|
2
|
+
|
|
3
|
+
# # We can't catch any Module errors here or else a whole string of errors will
|
|
4
|
+
# # get lost on it's way up to the PipeFill where this is imported if OTHER
|
|
5
|
+
# # depdendencies are missing, such as the torch module itself.
|
|
6
|
+
|
|
7
|
+
# __all__ = (
|
|
8
|
+
# "PyTorchRegressor",
|
|
9
|
+
# )
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import torch.nn as nn
|
|
2
|
+
|
|
3
|
+
# define the LSTM model
|
|
4
|
+
class LSTMAutoRegressive(nn.Module):
|
|
5
|
+
def __init__(self,
|
|
6
|
+
input_features,
|
|
7
|
+
hidden_dim,
|
|
8
|
+
output_features,
|
|
9
|
+
dropout=0.1,
|
|
10
|
+
lookback=8):
|
|
11
|
+
super(LSTMAutoRegressive, self).__init__()
|
|
12
|
+
self.hidden_dim = hidden_dim
|
|
13
|
+
self.input_net = nn.Sequential(
|
|
14
|
+
nn.Dropout(dropout), nn.Linear(input_features, hidden_dim)
|
|
15
|
+
)
|
|
16
|
+
self.lstm = nn.LSTM(hidden_dim, hidden_dim, num_layers=2,
|
|
17
|
+
batch_first=True, dropout=dropout)
|
|
18
|
+
|
|
19
|
+
self.output_layer = nn.Sequential(
|
|
20
|
+
nn.Linear(hidden_dim*lookback, output_features),
|
|
21
|
+
nn.Tanh()
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
def forward(self, input, hn=None, cn=None):
|
|
25
|
+
x = self.input_net(input)
|
|
26
|
+
if hn is None or cn is None:
|
|
27
|
+
x, (hn, cn) = self.lstm(x)
|
|
28
|
+
else:
|
|
29
|
+
x, (hn, cn) = self.lstm(x, (hn, cn))
|
|
30
|
+
x = x.reshape(-1, 1, input.shape[1]*x.shape[-1])
|
|
31
|
+
output = self.output_layer(x)
|
|
32
|
+
# only interested in the last output of the LSTM
|
|
33
|
+
# output = output[:, -1, :].unsqueeze(1) # , (hn, cn)
|
|
34
|
+
return output, hn, cn
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# define the LSTM model
|
|
38
|
+
class LSTMDirect(nn.Module):
|
|
39
|
+
def __init__(self,
|
|
40
|
+
input_features,
|
|
41
|
+
hidden_dim,
|
|
42
|
+
output_features,
|
|
43
|
+
dropout=0.1,
|
|
44
|
+
lookback=16,
|
|
45
|
+
num_layers=2):
|
|
46
|
+
super(LSTMDirect, self).__init__()
|
|
47
|
+
|
|
48
|
+
self.lstm = nn.LSTM(int(input_features), int(input_features), num_layers=num_layers,
|
|
49
|
+
batch_first=True, dropout=dropout)
|
|
50
|
+
|
|
51
|
+
self.output_net = nn.Sequential(
|
|
52
|
+
nn.Linear(int(input_features)*lookback, int(hidden_dim)),
|
|
53
|
+
nn.ReLU(),
|
|
54
|
+
nn.Dropout(dropout),
|
|
55
|
+
nn.Linear(int(hidden_dim), int(hidden_dim/2)),
|
|
56
|
+
nn.ReLU(),
|
|
57
|
+
nn.Dropout(dropout),
|
|
58
|
+
nn.Linear(int(hidden_dim/2), int(hidden_dim/4)),
|
|
59
|
+
nn.ReLU(),
|
|
60
|
+
nn.Dropout(dropout),
|
|
61
|
+
nn.Linear(int(hidden_dim/4), output_features)
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
def forward(self, input, hn=None, cn=None):
|
|
65
|
+
# x = self.input_net(input)
|
|
66
|
+
if hn is None or cn is None:
|
|
67
|
+
x, (hn, cn) = self.lstm(input)
|
|
68
|
+
else:
|
|
69
|
+
x, (hn, cn) = self.lstm(input, (hn, cn))
|
|
70
|
+
# flatten to a 1D vector space before decoding
|
|
71
|
+
x = x.reshape(-1, 1, input.shape[1]*x.shape[-1])
|
|
72
|
+
output = self.output_net(x)
|
|
73
|
+
|
|
74
|
+
return output, hn, cn
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Net(torch.nn.Module):
|
|
5
|
+
def __init__(self, input_features=32, output_features=6, output_dim=6):
|
|
6
|
+
super().__init__()
|
|
7
|
+
self.output_dim = output_dim
|
|
8
|
+
self.output_features = output_features
|
|
9
|
+
self.hidden_features = 1024
|
|
10
|
+
self.net = torch.nn.Sequential(
|
|
11
|
+
torch.nn.Linear(input_features, self.hidden_features),
|
|
12
|
+
torch.nn.ReLU(),
|
|
13
|
+
torch.nn.Linear(self.hidden_features, self.hidden_features),
|
|
14
|
+
torch.nn.ReLU(),
|
|
15
|
+
torch.nn.Linear(self.hidden_features,
|
|
16
|
+
output_features * output_dim),
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
def forward(self, x):
|
|
20
|
+
y = self.net(x)
|
|
21
|
+
return y
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class MLPBase(torch.nn.Module):
|
|
25
|
+
def __init__(self, input_features, output_features, dropout=0.1, hidden_dim=256):
|
|
26
|
+
super().__init__()
|
|
27
|
+
|
|
28
|
+
self.input_net = torch.nn.Sequential(
|
|
29
|
+
torch.nn.Linear(int(input_features), int(hidden_dim)),
|
|
30
|
+
torch.nn.ReLU(),
|
|
31
|
+
torch.nn.Dropout(dropout),
|
|
32
|
+
torch.nn.Linear(int(hidden_dim), int(hidden_dim/2)),
|
|
33
|
+
torch.nn.ReLU(),
|
|
34
|
+
torch.nn.Dropout(dropout),
|
|
35
|
+
torch.nn.Linear(int(hidden_dim/2), int(hidden_dim/4)),
|
|
36
|
+
torch.nn.ReLU(),
|
|
37
|
+
torch.nn.Dropout(dropout),
|
|
38
|
+
torch.nn.Linear(int(hidden_dim/4), int(output_features))
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def forward(self, x):
|
|
42
|
+
y = self.input_net(x)
|
|
43
|
+
# x = self.net(x)
|
|
44
|
+
# y = self.output_layer(x)
|
|
45
|
+
return y
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
|
|
2
|
+
import torch.nn as nn
|
|
3
|
+
import torch
|
|
4
|
+
from flowml.models.pytorch.utils import PositionalEncoding
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
The architecture is based on the paper “Attention Is All You Need”.
|
|
8
|
+
Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez,
|
|
9
|
+
Lukasz Kaiser, and Illia Polosukhin. 2017.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TransformerPositionalEncoding(nn.Module):
|
|
14
|
+
def __init__(self,
|
|
15
|
+
input_features=7,
|
|
16
|
+
output_features=7,
|
|
17
|
+
num_layers=2,
|
|
18
|
+
dropout=0.1,
|
|
19
|
+
hidden_dim=1024,
|
|
20
|
+
lookback=10,
|
|
21
|
+
nhead=8):
|
|
22
|
+
super(TransformerPositionalEncoding, self).__init__()
|
|
23
|
+
self.dim_val = input_features - (input_features % nhead)
|
|
24
|
+
|
|
25
|
+
# small dimension change to ensure input is divisible by nhead
|
|
26
|
+
self.input_net = nn.Sequential(
|
|
27
|
+
nn.Linear(input_features, self.dim_val)
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
# Positional encoding for sequences
|
|
31
|
+
self.positional_encoding = PositionalEncoding(
|
|
32
|
+
d_model=self.dim_val, max_len=self.dim_val)
|
|
33
|
+
self.encoder_layer = nn.TransformerEncoderLayer(
|
|
34
|
+
d_model=self.dim_val, dim_feedforward=self.dim_val, nhead=nhead,
|
|
35
|
+
dropout=dropout, batch_first=True)
|
|
36
|
+
self.transformer = nn.TransformerEncoder(self.encoder_layer, num_layers=num_layers)
|
|
37
|
+
|
|
38
|
+
self.output_net = nn.Sequential(
|
|
39
|
+
nn.Linear(self.dim_val*lookback, int(hidden_dim)),
|
|
40
|
+
nn.ReLU(),
|
|
41
|
+
nn.Dropout(dropout),
|
|
42
|
+
nn.Linear(int(hidden_dim), int(hidden_dim/2)),
|
|
43
|
+
nn.ReLU(),
|
|
44
|
+
nn.Dropout(dropout),
|
|
45
|
+
nn.Linear(int(hidden_dim/2), int(hidden_dim/4)),
|
|
46
|
+
nn.ReLU(),
|
|
47
|
+
nn.Dropout(dropout),
|
|
48
|
+
nn.Linear(int(hidden_dim/4), output_features)
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
def forward(self, input, mask=None, add_positional_encoding=True):
|
|
52
|
+
"""
|
|
53
|
+
Args:
|
|
54
|
+
x: Input features of shape [Batch, SeqLen, input_dim]
|
|
55
|
+
mask: Mask to apply on the attention outputs (optional)
|
|
56
|
+
add_positional_encoding: If True, we add the positional encoding to the input.
|
|
57
|
+
Might not be desired for some tasks.
|
|
58
|
+
"""
|
|
59
|
+
x = self.input_net(input)
|
|
60
|
+
if add_positional_encoding:
|
|
61
|
+
x = self.positional_encoding(x)
|
|
62
|
+
x = self.transformer(x, mask=mask)
|
|
63
|
+
# transform to a 1D vector space before decoding
|
|
64
|
+
x = x.reshape(-1, 1, input.shape[1]*x.shape[-1])
|
|
65
|
+
output = self.output_net(x)
|
|
66
|
+
|
|
67
|
+
return output
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# class TransformerTraditional(nn.Module):
|
|
71
|
+
# def __init__(self, feature_size=7, output_features=7, num_layers=2, dropout=0.1, dim_val=1024,
|
|
72
|
+
# input_dropout=0.1, lookback=10):
|
|
73
|
+
# super(TransformerTraditional, self).__init__()
|
|
74
|
+
# self.input_net = nn.Sequential(
|
|
75
|
+
# nn.Dropout(input_dropout), nn.Linear(feature_size, dim_val)
|
|
76
|
+
# )
|
|
77
|
+
# # Positional encoding for sequences
|
|
78
|
+
# self.positional_encoding = PositionalEncoding(d_model=dim_val, max_len=dim_val)
|
|
79
|
+
# self.encoder_layer = nn.TransformerEncoderLayer(
|
|
80
|
+
# d_model=dim_val, nhead=8, dropout=dropout, batch_first=True)
|
|
81
|
+
# self.transformer = nn.TransformerEncoder(self.encoder_layer, num_layers=num_layers)
|
|
82
|
+
# self.output_layer = nn.Linear(dim_val, output)
|
|
83
|
+
|
|
84
|
+
# def forward(self, x, lookback, mask=None, add_positional_encoding=True):
|
|
85
|
+
# """
|
|
86
|
+
# Args:
|
|
87
|
+
# x: Input features of shape [Batch, SeqLen, input_dim]
|
|
88
|
+
# mask: Mask to apply on the attention outputs (optional)
|
|
89
|
+
# add_positional_encoding: If True, we add the positional encoding to the input.
|
|
90
|
+
# Might not be desired for some tasks.
|
|
91
|
+
# """
|
|
92
|
+
# x = self.input_net(x)
|
|
93
|
+
# if add_positional_encoding:
|
|
94
|
+
# x = self.positional_encoding(x)
|
|
95
|
+
# x = self.transformer(x, mask=mask)
|
|
96
|
+
# x = self.output_layer(x)
|
|
97
|
+
# return x
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# class Transformer(nn.Module):
|
|
101
|
+
# # d_model : number of features
|
|
102
|
+
# def __init__(self, feature_size=7, output=7, num_layers=2, dropout=0, dim_val=1024):
|
|
103
|
+
# super(Transformer, self).__init__()
|
|
104
|
+
# # Creating the three linear layers needed for the model
|
|
105
|
+
# self.encoder_input_layer = nn.Linear(
|
|
106
|
+
# in_features=feature_size,
|
|
107
|
+
# out_features=dim_val
|
|
108
|
+
# )
|
|
109
|
+
|
|
110
|
+
# self.linear_mapping = nn.Linear(
|
|
111
|
+
# in_features=dim_val,
|
|
112
|
+
# out_features=output
|
|
113
|
+
# )
|
|
114
|
+
|
|
115
|
+
# self.encoder_layer = nn.TransformerEncoderLayer(d_model=dim_val, nhead=8, dropout=dropout)
|
|
116
|
+
# self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=num_layers)
|
|
117
|
+
# self.decoder = nn.Linear(feature_size, output)
|
|
118
|
+
# self.init_weights()
|
|
119
|
+
|
|
120
|
+
# def init_weights(self):
|
|
121
|
+
# initrange = 0.1
|
|
122
|
+
# self.decoder.bias.data.zero_()
|
|
123
|
+
# self.decoder.weight.data.uniform_(-initrange, initrange)
|
|
124
|
+
|
|
125
|
+
# def _generate_square_subsequent_mask(self, sz):
|
|
126
|
+
# mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
|
|
127
|
+
# mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
|
|
128
|
+
# return mask
|
|
129
|
+
|
|
130
|
+
# def forward(self, src): # , device):
|
|
131
|
+
|
|
132
|
+
# src = self.encoder_input_layer(src)
|
|
133
|
+
# # mask = self._generate_square_subsequent_mask(len(src)).to(device)
|
|
134
|
+
# output = self.transformer_encoder(src) # , mask)
|
|
135
|
+
# output = self.linear_mapping(output)
|
|
136
|
+
# # output = self.decoder(output)
|
|
137
|
+
# return output
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from typing import Any, Callable
|
|
3
|
+
from abc import abstractmethod
|
|
4
|
+
from io import BytesIO
|
|
5
|
+
|
|
6
|
+
from flowml.models.base import BaseMLModel
|
|
7
|
+
from flowdapt.compute.artifacts import Artifact
|
|
8
|
+
from flowml.models.pytorch.utils import check_for_gpu, set_num_threads
|
|
9
|
+
from flowdapt.lib.logger import get_logger
|
|
10
|
+
# from flowml.tensorboard import TensorboardLogger
|
|
11
|
+
import pandas as pd
|
|
12
|
+
import numpy.typing as npt
|
|
13
|
+
from torch.utils.tensorboard import SummaryWriter
|
|
14
|
+
# from datetime import datetime
|
|
15
|
+
from flowml.models.pytorch.architectures.mlp import Net
|
|
16
|
+
import numpy as np
|
|
17
|
+
import tempfile
|
|
18
|
+
import os
|
|
19
|
+
|
|
20
|
+
logger = get_logger(__name__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class BasePyTorchModel(BaseMLModel):
|
|
24
|
+
def __init_model__(
|
|
25
|
+
self,
|
|
26
|
+
epochs: int = 10,
|
|
27
|
+
batch_size: int = 32,
|
|
28
|
+
lookback: int = 10,
|
|
29
|
+
hidden_dim: int = 32,
|
|
30
|
+
num_layers: int = 2,
|
|
31
|
+
dropout: float = 0.1,
|
|
32
|
+
nhead: int = 8,
|
|
33
|
+
shuffle: bool = False,
|
|
34
|
+
extra_train_params: dict[str, Any] = {},
|
|
35
|
+
logger: Callable | None = None,
|
|
36
|
+
early_stopping: bool = False,
|
|
37
|
+
num_threads: int = 1,
|
|
38
|
+
**kwargs
|
|
39
|
+
) -> "BasePyTorchModel":
|
|
40
|
+
self.num_threads = num_threads
|
|
41
|
+
self._device = check_for_gpu()
|
|
42
|
+
set_num_threads(num_threads)
|
|
43
|
+
self.epochs = epochs
|
|
44
|
+
self.batch_size = batch_size
|
|
45
|
+
self.lookback = lookback
|
|
46
|
+
self.shuffle = shuffle
|
|
47
|
+
self.hidden_dim = hidden_dim
|
|
48
|
+
self.num_layers = num_layers
|
|
49
|
+
self.dropout = dropout
|
|
50
|
+
self.nhead = nhead
|
|
51
|
+
self._reset_logger = True
|
|
52
|
+
self.extra_train_params = extra_train_params
|
|
53
|
+
if self.default_logger:
|
|
54
|
+
self.logger = SummaryWriter
|
|
55
|
+
else:
|
|
56
|
+
self.logger = logger
|
|
57
|
+
self.tb_logger = None
|
|
58
|
+
self.optimizer: torch.optim.Optimizer | None = None # torch.optim.Adam()
|
|
59
|
+
self.nn: torch.nn.Module = Net()
|
|
60
|
+
self.best_loss: float | None = None
|
|
61
|
+
self.hn = None
|
|
62
|
+
self.cn = None
|
|
63
|
+
self.early_stopping = early_stopping
|
|
64
|
+
|
|
65
|
+
return self
|
|
66
|
+
|
|
67
|
+
def get_params(self):
|
|
68
|
+
return {
|
|
69
|
+
"epochs": self.epochs,
|
|
70
|
+
"batch_size": self.batch_size,
|
|
71
|
+
"lookback": self.lookback,
|
|
72
|
+
"hidden_dim": self.hidden_dim,
|
|
73
|
+
"shuffle": self.shuffle,
|
|
74
|
+
"num_layers": self.num_layers,
|
|
75
|
+
"dropout": self.dropout,
|
|
76
|
+
"nhead": self.nhead,
|
|
77
|
+
"extra_train_params": self.extra_train_params,
|
|
78
|
+
"default_logger": False,
|
|
79
|
+
"num_threads": self.num_threads
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
# This is required for the model to be added to ClusterMemory as-is,
|
|
83
|
+
# while keeping it portable to be trained on a GPU but used on a CPU
|
|
84
|
+
# if needed. ClusterMemory *ALWAYS* uses some type of pickling for over
|
|
85
|
+
# the wire communication.
|
|
86
|
+
def __getstate__(self):
|
|
87
|
+
buffer = BytesIO()
|
|
88
|
+
self.logger = None
|
|
89
|
+
# Serialize the state
|
|
90
|
+
state = {
|
|
91
|
+
"params": self.get_params(),
|
|
92
|
+
"name": self.name_str,
|
|
93
|
+
"best_loss": self.best_loss,
|
|
94
|
+
"namespace": self._namespace,
|
|
95
|
+
"network_type": self.nn,
|
|
96
|
+
"optimizer_type": self.optimizer,
|
|
97
|
+
# "network_statedict": self.nn.state_dict() if self.nn else None,
|
|
98
|
+
# "optimizer_statedict": self.optimizer.state_dict() if self.optimizer else None,
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
# Use torch save so we know it's compatible with the model
|
|
102
|
+
torch.save(state, buffer)
|
|
103
|
+
return buffer.getvalue()
|
|
104
|
+
|
|
105
|
+
def __setstate__(self, bytear):
|
|
106
|
+
# Load the state and make sure it's loaded on CPU
|
|
107
|
+
buffer = BytesIO(bytear)
|
|
108
|
+
|
|
109
|
+
_device = check_for_gpu()
|
|
110
|
+
|
|
111
|
+
if _device == torch.device("cpu"):
|
|
112
|
+
state = torch.load(buffer, map_location=_device)
|
|
113
|
+
else:
|
|
114
|
+
state = torch.load(buffer)
|
|
115
|
+
|
|
116
|
+
set_num_threads(state["params"]["num_threads"])
|
|
117
|
+
# Reinitialize the model
|
|
118
|
+
self.__init__(state["name"], state["namespace"], **state["params"])
|
|
119
|
+
|
|
120
|
+
self.nn = state["network_type"]
|
|
121
|
+
self.optimizer = state["optimizer_type"]
|
|
122
|
+
|
|
123
|
+
if _device == torch.device("cuda"):
|
|
124
|
+
self.nn.to(_device)
|
|
125
|
+
|
|
126
|
+
# network_statedict = state["network_statedict"]
|
|
127
|
+
# optimizer_statedict = state["optimizer_statedict"]
|
|
128
|
+
|
|
129
|
+
# self.nn.load_state_dict(network_statedict)
|
|
130
|
+
# self.optimizer.load_state_dict(optimizer_statedict)
|
|
131
|
+
|
|
132
|
+
self.best_loss = state["best_loss"]
|
|
133
|
+
|
|
134
|
+
@classmethod
|
|
135
|
+
def from_artifact(cls, **kwargs) -> Callable[[Artifact], Any]:
|
|
136
|
+
def _inner(artifact: Artifact):
|
|
137
|
+
# Require model artifact
|
|
138
|
+
if not artifact.get_meta("type", None) == "model":
|
|
139
|
+
raise ValueError("Artifact must be of type `model`")
|
|
140
|
+
|
|
141
|
+
# Get the meta from the Artifact
|
|
142
|
+
model_params = artifact.get_meta("params", {})
|
|
143
|
+
study_id = artifact.get_meta("namespace", None)
|
|
144
|
+
|
|
145
|
+
# Update the model params with the ones from the artifact
|
|
146
|
+
kwargs.update(model_params)
|
|
147
|
+
file = artifact.get_file("model.pt")
|
|
148
|
+
|
|
149
|
+
_device = check_for_gpu()
|
|
150
|
+
set_num_threads(model_params["num_threads"])
|
|
151
|
+
|
|
152
|
+
# Create the instance and assign the model
|
|
153
|
+
instance = cls(name=artifact.name, **model_params)
|
|
154
|
+
|
|
155
|
+
# Load the model and optimizer
|
|
156
|
+
with file.open(mode="rb") as f:
|
|
157
|
+
if _device == torch.device("cpu"):
|
|
158
|
+
checkpoint = torch.load(f, map_location=_device)
|
|
159
|
+
else:
|
|
160
|
+
checkpoint = torch.load(f)
|
|
161
|
+
|
|
162
|
+
instance.nn = checkpoint["network_architecture"]
|
|
163
|
+
instance.optimizer = checkpoint["optimizer"]
|
|
164
|
+
|
|
165
|
+
# instance.nn.load_state_dict(checkpoint["model_state_dict"])
|
|
166
|
+
# instance.optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
|
|
167
|
+
instance.best_loss = checkpoint["best_loss"]
|
|
168
|
+
|
|
169
|
+
if _device == torch.device("cuda"):
|
|
170
|
+
instance.nn.to(_device)
|
|
171
|
+
instance._device = _device
|
|
172
|
+
instance._namespace = study_id
|
|
173
|
+
|
|
174
|
+
return instance
|
|
175
|
+
return _inner
|
|
176
|
+
|
|
177
|
+
def to_artifact(self, **kwargs) -> Callable[[Artifact, Any], None]:
|
|
178
|
+
def _inner(artifact: Artifact, value: Any):
|
|
179
|
+
assert self.nn and self.optimizer, "Model must be trained before saving"
|
|
180
|
+
|
|
181
|
+
# Add the meta to the artifact
|
|
182
|
+
artifact["type"] = "model"
|
|
183
|
+
artifact["params"] = self.get_params()
|
|
184
|
+
artifact["model_type"] = self.get_type()
|
|
185
|
+
artifact["namespace"] = self._namespace
|
|
186
|
+
|
|
187
|
+
with artifact.new_file("model.pt", exist_ok=True).open("wb") as f:
|
|
188
|
+
torch.save(
|
|
189
|
+
{
|
|
190
|
+
"network_architecture": self.nn,
|
|
191
|
+
"best_loss": self.best_loss,
|
|
192
|
+
"optimizer": self.optimizer,
|
|
193
|
+
# "model_state_dict": self.nn.state_dict(),
|
|
194
|
+
# "optimizer_state_dict": self.optimizer.state_dict()
|
|
195
|
+
},
|
|
196
|
+
f
|
|
197
|
+
)
|
|
198
|
+
return _inner
|
|
199
|
+
|
|
200
|
+
@abstractmethod
|
|
201
|
+
def fit(self, *args, **kwargs):
|
|
202
|
+
...
|
|
203
|
+
|
|
204
|
+
@abstractmethod
|
|
205
|
+
def predict(self, *args, **kwargs):
|
|
206
|
+
...
|
|
207
|
+
|
|
208
|
+
def log(self, name: str, value: Any, step: int):
|
|
209
|
+
"""
|
|
210
|
+
Wrapper for the logger callaback
|
|
211
|
+
"""
|
|
212
|
+
if self.logger is None:
|
|
213
|
+
return
|
|
214
|
+
|
|
215
|
+
if self._reset_logger:
|
|
216
|
+
self.current_log_dir = os.path.join(tempfile.gettempdir(), self.name_str)
|
|
217
|
+
os.makedirs(self.current_log_dir, exist_ok=True)
|
|
218
|
+
self.tb_logger = self.logger(self.current_log_dir)
|
|
219
|
+
self._reset_logger = False
|
|
220
|
+
|
|
221
|
+
self.tb_logger.add_scalar(name, value, step)
|
|
222
|
+
|
|
223
|
+
def reset_logger(self):
|
|
224
|
+
if self.logger is None:
|
|
225
|
+
return
|
|
226
|
+
self.tb_logger.flush()
|
|
227
|
+
self.tb_logger.close()
|
|
228
|
+
self.logger_to_artifact()
|
|
229
|
+
self.tb_logger = None
|
|
230
|
+
self._reset_logger = True
|
|
231
|
+
|
|
232
|
+
def _validate_args(self,
|
|
233
|
+
xs: npt.ArrayLike | pd.DataFrame,
|
|
234
|
+
ys: npt.ArrayLike | pd.DataFrame | None = None,
|
|
235
|
+
eval_set: Any = None
|
|
236
|
+
) -> tuple[torch.Tensor, torch.Tensor | None,
|
|
237
|
+
torch.Tensor | None, torch.Tensor | None]:
|
|
238
|
+
"""
|
|
239
|
+
Validates the incoming arguments to ensure they are numpy arrays
|
|
240
|
+
"""
|
|
241
|
+
if isinstance(xs, pd.DataFrame):
|
|
242
|
+
x = torch.from_numpy(xs.values).float() # xs.to_numpy()
|
|
243
|
+
logger.info(f"Validating {x.shape[0]} samples and {x.shape[1]} features")
|
|
244
|
+
elif xs is None:
|
|
245
|
+
x = None
|
|
246
|
+
elif type(xs) == npt.ArrayLike or type(xs) == np.ndarray:
|
|
247
|
+
x = torch.from_numpy(xs).float()
|
|
248
|
+
else:
|
|
249
|
+
raise ValueError("xs must be a numpy array or pandas dataframe")
|
|
250
|
+
|
|
251
|
+
if isinstance(ys, pd.DataFrame):
|
|
252
|
+
y = torch.from_numpy(ys.values).float() # .to_numpy()
|
|
253
|
+
elif ys is None:
|
|
254
|
+
y = None
|
|
255
|
+
elif type(ys) == npt.ArrayLike or type(ys) == np.ndarray:
|
|
256
|
+
y = torch.from_numpy(ys).float()
|
|
257
|
+
else:
|
|
258
|
+
raise ValueError("ys must be a numpy array or pandas dataframe")
|
|
259
|
+
|
|
260
|
+
if eval_set is not None and isinstance(eval_set[0][0], pd.DataFrame):
|
|
261
|
+
valid_xs = torch.from_numpy(eval_set[0][0].values).float() # to_numpy()
|
|
262
|
+
valid_ys = torch.from_numpy(eval_set[0][1].values).float()
|
|
263
|
+
elif eval_set is None:
|
|
264
|
+
valid_xs = None
|
|
265
|
+
valid_ys = None
|
|
266
|
+
elif type(eval_set[0][0]) == npt.ArrayLike or type(eval_set[0][0]) == np.ndarray:
|
|
267
|
+
valid_xs = torch.from_numpy(eval_set[0][0]).float()
|
|
268
|
+
valid_ys = torch.from_numpy(eval_set[0][1]).float()
|
|
269
|
+
else:
|
|
270
|
+
raise ValueError("eval_set must be a numpy array or pandas dataframe")
|
|
271
|
+
|
|
272
|
+
return x, y, valid_xs, valid_ys
|