iapytoo 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,35 @@
1
+ import os
2
+ import mlflow
3
+
4
+
5
+ class Context:
6
+
7
+ filename = "context.txt"
8
+
9
+ def __init__(self, run_id=None) -> None:
10
+ self.run_id = run_id
11
+ self.last_epoch = -1
12
+ self.epoch = -1
13
+ if run_id is not None:
14
+ self._load_from_artifact(run_id)
15
+
16
+ def _load_from_artifact(self, run_id):
17
+ run = mlflow.get_run(self.run_id)
18
+ assert run is not None, f"unable to find run {self.run_id}"
19
+
20
+ local_path = mlflow.artifacts.download_artifacts(
21
+ run_id=run_id, artifact_path=Context.filename
22
+ )
23
+ with open(local_path, "r") as f:
24
+ for line in f:
25
+ key, value = line.strip().split(": ", 1)
26
+ if key == "epoch":
27
+ self.last_epoch = int(value)
28
+
29
+ def save(self, dirname):
30
+ filepath = os.path.join(dirname, Context.filename)
31
+ context_dict = {"epoch": self.epoch}
32
+ with open(filepath, "w") as f:
33
+ for key, value in context_dict.items():
34
+ f.write(f"{key}: {value}\n")
35
+ return filepath
@@ -0,0 +1,215 @@
1
+ import torch.nn as nn
2
+ from iapytoo.utils.meta_singleton import MetaSingleton
3
+ import torch.optim as to
4
+ from torch.optim.lr_scheduler import StepLR
5
+
6
+
7
+ class ModelError(Exception):
8
+ def __init__(self, *args: object) -> None:
9
+ super().__init__(*args)
10
+
11
+
12
+ class WeightInitiator:
13
+ def __call__(self, m):
14
+ """method to be overloaded for weight initialization by use of 'model.apply(initiator)'"""
15
+ classname = m.__class__.__name__
16
+ print(f"init {classname}")
17
+
18
+
19
+ class Model(nn.Module):
20
+ def __init__(self, loader, config) -> None:
21
+ super().__init__()
22
+
23
+ def weight_initiator(self):
24
+ """return an WeightInitiator subclass that will be used to initialize weights of this model"""
25
+ return None
26
+
27
+ def predict(self, model_output):
28
+ """This method gives the opportunity to transform output of the model to something equivalent to Y given by the dataset
29
+ For example, class from probability distribution"""
30
+ return model_output
31
+
32
+
33
+ class ModelFactory(metaclass=MetaSingleton):
34
+ def __init__(self) -> None:
35
+ self.models_dict = {}
36
+
37
+ def register_model(self, key, model_cls):
38
+ self.models_dict[key] = model_cls
39
+
40
+ def create_model(self, kind: str, config: dict, loader, device="cpu"):
41
+ """Creates an architecture of NN
42
+
43
+ Args:
44
+ kind (str): kind of NN, key for the factory
45
+ config (dict): config dict to use to initialize model
46
+ loader (DataLoader): a dataloader to have input/output dimensions
47
+ device (str, optional): Defaults to "cpu".
48
+
49
+ Raises:
50
+ ModelError: error raised if no architecture fit kind key
51
+
52
+ Returns:
53
+ nn.Module: pytorch model
54
+ """
55
+ try:
56
+ model: Model = self.models_dict[kind](loader, config)
57
+ except KeyError:
58
+ raise ModelError(f"model {kind} is not handled")
59
+
60
+ initiator = model.weight_initiator()
61
+ if initiator is not None:
62
+ model.apply(initiator)
63
+
64
+ model = model.to(device=device)
65
+ return model
66
+
67
+
68
+ class OptimizerError(Exception):
69
+ def __init__(self, *args: object) -> None:
70
+ super().__init__(*args)
71
+
72
+
73
+ class Optimizer:
74
+ def __init__(self, model, config) -> None:
75
+ self.torch_optimizer = None
76
+
77
+
78
+ class AdamOptimizer(Optimizer):
79
+ def __init__(self, model, config) -> None:
80
+ super().__init__(model, config)
81
+
82
+ kwargs = {"lr": config["learning_rate"]}
83
+ if "weight_decay" in config:
84
+ kwargs["weight_decay"] = config["weight_decay"]
85
+ if "betas" in config:
86
+ kwargs["betas"] = config["betas"]
87
+
88
+ self.torch_optimizer = to.Adam(model.parameters(), **kwargs)
89
+
90
+
91
+ class RMSpropOptimizer(Optimizer):
92
+ def __init__(self, model, config):
93
+ super().__init__(model, config)
94
+ self.torch_optimizer = to.RMSprop(
95
+ model.parameters(), lr=config["learning_rate"]
96
+ )
97
+
98
+
99
+ class SGDOptimizer(Optimizer):
100
+ def __init__(self, model, config) -> None:
101
+ super().__init__(model, config)
102
+ kwargs = {"lr": config["learning_rate"]}
103
+ if "weight_decay" in config:
104
+ kwargs["weight_decay"] = config["weight_decay"]
105
+ if "momentum" in config:
106
+ kwargs["momentum"] = config["momentum"]
107
+ self.torch_optimizer = to.SGD(model.parameters(), **kwargs)
108
+
109
+
110
+ class OptimizerFactory(metaclass=MetaSingleton):
111
+ def __init__(self) -> None:
112
+ self.optimizers_dict = {
113
+ "adam": AdamOptimizer,
114
+ "sgd": SGDOptimizer,
115
+ "rmsprop": RMSpropOptimizer,
116
+ }
117
+
118
+ def register_optimizer(self, key, optimizer_cls):
119
+ self.optimizers_dict[key] = optimizer_cls
120
+
121
+ def create_optimizer(self, kind: str, model, config: dict):
122
+ """Creates an optimizer for the NN
123
+
124
+ Args:
125
+ kind (str): kind of optimizer, key for the factory
126
+ config (dict): config dict to use to initialize optimizer
127
+
128
+ Raises:
129
+ OptimizerError: error raised if no architecture fit kind key
130
+
131
+ Returns:
132
+ nn.Module: pytorch optimizer
133
+ """
134
+ try:
135
+ optimizer = self.optimizers_dict[kind](model, config)
136
+ except KeyError:
137
+ raise OptimizerError(f"optimizer {kind} is not handled")
138
+
139
+ return optimizer
140
+
141
+
142
+ class SchedulerError(Exception):
143
+ def __init__(self, *args: object) -> None:
144
+ super().__init__(*args)
145
+
146
+
147
+ class Scheduler:
148
+ def __init__(self, optimizer, config) -> None:
149
+ self.lr_scheduler = None
150
+
151
+
152
+ class StepScheduler(Scheduler):
153
+ def __init__(self, optimizer, config) -> None:
154
+ super().__init__(optimizer, config)
155
+
156
+ kwargs = {}
157
+ kwargs["gamma"] = config.get("gamma", 0.7)
158
+ kwargs["step_size"] = config.get("step_size", 1)
159
+ self.lr_scheduler = StepLR(optimizer, **kwargs)
160
+
161
+
162
+ class SchedulerFactory(metaclass=MetaSingleton):
163
+ def __init__(self) -> None:
164
+ self.schedulers_dict = {"step": StepScheduler}
165
+
166
+ def register_scheduler(self, key, scheduler_cls):
167
+ self.schedulers_dict[key] = scheduler_cls
168
+
169
+ def create_scheduler(self, kind: str, optimizer, config: dict):
170
+ """Creates an learning rate scheduler
171
+
172
+ Args:
173
+ kind (str): kind of scheduler, key for the factory
174
+ config (dict): config dict to use to initialize scheduler
175
+
176
+ Raises:
177
+ SchedulerError: error raised if no architecture fit kind key
178
+
179
+ Returns:
180
+ nn.Module: pytorch scheduler
181
+ """
182
+ try:
183
+ scheduler = self.schedulers_dict[kind](optimizer, config)
184
+ except KeyError:
185
+ raise SchedulerError(f"scheduler {kind} is not handled")
186
+
187
+ return scheduler
188
+
189
+
190
+ class LossFactory(metaclass=MetaSingleton):
191
+ def __init__(self) -> None:
192
+ self.loss_dict = {
193
+ "mse": nn.MSELoss,
194
+ "nll": nn.NLLLoss,
195
+ "cel": nn.CrossEntropyLoss,
196
+ }
197
+
198
+ def register_loss(self, key, loss_cls):
199
+ self.loss_dict[key] = loss_cls
200
+
201
+ def create_loss(self, kind: str):
202
+ """Creates a loss criterion
203
+
204
+ Args:
205
+ kind (str): kind of loss, key for the factory
206
+
207
+ Returns:
208
+ nn.Module: pytorch loss function
209
+ """
210
+ try:
211
+ criterion = self.loss_dict[kind]()
212
+ except KeyError:
213
+ raise Exception(f"loss {kind} is not handled")
214
+
215
+ return criterion
@@ -0,0 +1,190 @@
1
+ import os
2
+ import torch
3
+ import torch.nn as nn
4
+
5
+ import uuid
6
+ import tempfile
7
+ import numpy as np
8
+ import matplotlib.pyplot as plt
9
+ import matplotlib
10
+ import logging
11
+
12
+ from threading import Lock
13
+
14
+ import mlflow
15
+ from mlflow.models import ModelSignature
16
+ from mlflow.types.schema import TensorSpec, Schema
17
+
18
+ from iapytoo.utils.config import Config
19
+ from iapytoo.utils.display import lrfind_plot
20
+ from iapytoo.train.checkpoint import CheckPoint
21
+ from iapytoo.train.context import Context
22
+ from iapytoo.predictions import Predictions
23
+
24
+
25
+ class Logger:
26
+ @staticmethod
27
+ def _run_name(root_name):
28
+ return f"{root_name}_{str(uuid.uuid1())[:8]}"
29
+
30
+ @property
31
+ def experiment_id(self):
32
+ experiment = mlflow.get_experiment_by_name(self.config["project"])
33
+ if experiment is None:
34
+ return mlflow.create_experiment(self.config["project"])
35
+ else:
36
+ return experiment.experiment_id
37
+
38
+ def active_run_name(self):
39
+ active_run = mlflow.active_run()
40
+ if active_run:
41
+ return active_run.info.run_name
42
+
43
+ return None
44
+
45
+ @property
46
+ def config(self):
47
+ return self._config.__dict__
48
+
49
+ def __init__(self, config: Config, run_id: str = None) -> None:
50
+ self._config = config
51
+ self.run_id = run_id
52
+ self.agg = matplotlib.rcParams["backend"]
53
+ self.signature = None
54
+ self.lock = Lock()
55
+ print("tracking_uri", self.config["tracking_uri"])
56
+ if "tracking_uri" in self.config and self.config["tracking_uri"] is not None:
57
+ logging.info(f".. set tracking uri to {self.config['tracking_uri']}")
58
+ mlflow.set_tracking_uri(self.config["tracking_uri"])
59
+
60
+ self.context = Context(run_id)
61
+
62
+ def __enter__(self):
63
+ self.start()
64
+ return self
65
+
66
+ def __exit__(self, exc_type, exc_value, traceback):
67
+ self.close()
68
+
69
+ def start(self):
70
+ matplotlib.use("agg")
71
+ params_flag = self.run_id is None
72
+ active_run = mlflow.start_run(
73
+ experiment_id=self.experiment_id,
74
+ run_id=self.run_id,
75
+ run_name=self._run_name(self.config["run"]),
76
+ )
77
+ print("active_run", active_run.info.run_id)
78
+ self.run_id = active_run.info.run_id
79
+ if params_flag:
80
+ mlflow.log_params(self._params())
81
+
82
+ def close(self):
83
+ matplotlib.use(self.agg)
84
+ mlflow.end_run()
85
+
86
+ def set_signature(self, loader):
87
+ X, Y = next(iter(loader))
88
+ x_shape = list(X.shape)
89
+ x_shape[0] = -1
90
+ y_shape = list(Y.shape)
91
+ y_shape[0] = -1
92
+
93
+ input_schema = Schema([TensorSpec(type=np.dtype(np.float32), shape=x_shape)])
94
+ output_schema = Schema([TensorSpec(type=np.dtype(np.float32), shape=y_shape)])
95
+ self.signature = ModelSignature(inputs=input_schema, outputs=output_schema)
96
+
97
+ def _params(self):
98
+ # take care, some config parameters are saved by mlflow.
99
+ # When you run it again, these parameters can not change between two runs.
100
+ params = self._config.__dict__.copy()
101
+ if "run_id" in params:
102
+ del params["run_id"]
103
+ del params["epochs"]
104
+ del params["tracking_uri"]
105
+ return params
106
+
107
+ def __str__(self):
108
+ msg = "\nLogger:\n"
109
+ network = (
110
+ self.config["model"]
111
+ if "model" in self.config
112
+ else f"{self.config['discriminator']} & {self.config['generator']}"
113
+ )
114
+ msg += f"Network: {network}\n"
115
+ msg += f"matplotlib backend: {matplotlib.rcParams['backend']}, interactive: {matplotlib.is_interactive()}\n"
116
+ msg += f"tracking_uri: {mlflow.get_tracking_uri()}\n"
117
+ active_run = mlflow.active_run()
118
+ if active_run:
119
+ msg += f"Name: {active_run.info.run_name}\n"
120
+ msg += f"Experiment_id: {active_run.info.experiment_id}\n"
121
+ msg += f"Run_id: {self.run_id}\n"
122
+ if self.signature is not None:
123
+ msg += f"Signature {str(self.signature)}\n"
124
+
125
+ return msg
126
+
127
+ def summary(self):
128
+ logging.info(str(self))
129
+ logging.info(self._config)
130
+
131
+ def set_epoch(self, epoch):
132
+ self.context.epoch = epoch
133
+ self._save_context()
134
+
135
+ def _save_context(self):
136
+ with tempfile.TemporaryDirectory() as tmpdirname:
137
+ ctx_path = self.context.save(tmpdirname)
138
+ mlflow.log_artifact(ctx_path)
139
+
140
+ def log_checkpoint(self, checkpoint: CheckPoint):
141
+ with tempfile.TemporaryDirectory() as tmpdirname:
142
+ ckp_name = os.path.join(tmpdirname, "checkpoint.pt")
143
+ torch.save(checkpoint.params, ckp_name)
144
+ mlflow.log_artifact(local_path=ckp_name, artifact_path="checkpoints")
145
+
146
+ def save_model(self, model: nn.Module):
147
+ with self.lock:
148
+ # model for deployment don't need a GPU device
149
+ # store it on gpu
150
+ device = torch.device("cpu")
151
+ mlflow.pytorch.log_model(
152
+ model.to(device),
153
+ "model",
154
+ signature=self.signature,
155
+ extra_pip_requirements=["--extra-index-url https://zwergon.github.io"],
156
+ )
157
+
158
+ def can_report(self):
159
+ return self.context.epoch > self.context.last_epoch
160
+
161
+ def report_metric(self, epoch, metrics: dict):
162
+
163
+ with self.lock:
164
+ mlflow.log_metrics(metrics, step=epoch)
165
+
166
+ def report_metrics(self, epoch, metrics):
167
+
168
+ with self.lock:
169
+ values = {}
170
+ for k, v in metrics.results.items():
171
+ if len(v.shape) == 0:
172
+ values[k] = v.item()
173
+ else:
174
+ for c in range(v.shape[0]):
175
+ values[f"{k}_{c}"] = v[c].item()
176
+ mlflow.log_metrics(values, step=epoch)
177
+
178
+ def report_prediction(self, epoch, predictions: Predictions):
179
+ with self.lock:
180
+ plots = predictions.plot(epoch)
181
+ for name, fig in plots.items():
182
+ if fig is not None:
183
+ mlflow.log_figure(fig, artifact_file=f"{name}/{name}_{epoch}.png")
184
+ plt.close(fig)
185
+
186
+ def report_findlr(self, lrs, losses):
187
+ fig = lrfind_plot(lrs, losses)
188
+ with self.lock:
189
+ mlflow.log_figure(figure=fig, artifact_file="find_lr.jpg")
190
+ plt.close(fig)
iapytoo/train/loss.py ADDED
@@ -0,0 +1,33 @@
1
+ from typing import Any
2
+ from iapytoo.utils.iterative_mean import Mean
3
+
4
+
5
+ class Loss:
6
+ def __init__(self, n_losses=2) -> None:
7
+ self.n_losses = n_losses
8
+ self.losses = []
9
+
10
+ def flush(self):
11
+ for l in self.losses:
12
+ l.flush()
13
+
14
+ def __call__(self, index):
15
+ try:
16
+ return self.losses[index]
17
+ except IndexError:
18
+ raise Exception(f"Index {index} of loss out of range")
19
+
20
+ def reset(self):
21
+ self.losses = [Mean.create("ewm") for _ in range(self.n_losses)]
22
+
23
+ def state_dict(self):
24
+ state_dict = {
25
+ f"loss_{i}": self.losses[i].state_dict() for i in range(self.n_losses)
26
+ }
27
+ state_dict['n_losses'] = self.n_losses
28
+ return state_dict
29
+
30
+ def load_state_dict(self, state_dict):
31
+ self.n_losses = state_dict['n_losses']
32
+ for i in range(self.n_losses):
33
+ self.losses[i].load_state_dict(state_dict[f"loss_{i}"])