CarvusTrain 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- CarvusTrain/__init__.py +220 -0
- CarvusTrain/activation.py +193 -0
- CarvusTrain/callbacks.py +146 -0
- CarvusTrain/cli.py +309 -0
- CarvusTrain/configuration.py +183 -0
- CarvusTrain/constants.py +114 -0
- CarvusTrain/dataset.py +143 -0
- CarvusTrain/evaluation.py +100 -0
- CarvusTrain/exceptions.py +65 -0
- CarvusTrain/exporter.py +130 -0
- CarvusTrain/inference.py +1803 -0
- CarvusTrain/layers.py +359 -0
- CarvusTrain/logger.py +131 -0
- CarvusTrain/losses.py +172 -0
- CarvusTrain/memory.py +989 -0
- CarvusTrain/metrics.py +149 -0
- CarvusTrain/model.py +478 -0
- CarvusTrain/optimizer.py +273 -0
- CarvusTrain/parser.py +290 -0
- CarvusTrain/postprocessing.py +110 -0
- CarvusTrain/preprocessing.py +130 -0
- CarvusTrain/scheduler.py +123 -0
- CarvusTrain/tokenizer.py +338 -0
- CarvusTrain/trainer.py +409 -0
- CarvusTrain/utils.py +187 -0
- CarvusTrain/version.py +15 -0
- carvustrain-1.0.0.dist-info/METADATA +184 -0
- carvustrain-1.0.0.dist-info/RECORD +31 -0
- carvustrain-1.0.0.dist-info/WHEEL +4 -0
- carvustrain-1.0.0.dist-info/entry_points.txt +3 -0
- carvustrain-1.0.0.dist-info/licenses/LICENSE +21 -0
CarvusTrain/__init__.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""CarvusTrain - High Performance AI Model Training and Inference Framework."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Dict, List, Optional, Union
|
|
4
|
+
|
|
5
|
+
from carvustrain import activation, callbacks, layers, losses, metrics
|
|
6
|
+
from carvustrain.configuration import (
|
|
7
|
+
CarvusConfig,
|
|
8
|
+
DatasetConfig,
|
|
9
|
+
InferenceConfig,
|
|
10
|
+
ModelConfig,
|
|
11
|
+
TrainingConfig,
|
|
12
|
+
load_config_file,
|
|
13
|
+
)
|
|
14
|
+
from carvustrain.dataset import DataLoader, Dataset, TabularDataset, TextDataset
|
|
15
|
+
from carvustrain.evaluation import Benchmarker, Evaluator
|
|
16
|
+
from carvustrain.exceptions import (
|
|
17
|
+
CarvusTrainError,
|
|
18
|
+
ConfigurationError,
|
|
19
|
+
DatasetNotFoundError,
|
|
20
|
+
ExportError,
|
|
21
|
+
InferenceError,
|
|
22
|
+
ModelError,
|
|
23
|
+
ParserError,
|
|
24
|
+
TokenizerError,
|
|
25
|
+
TrainingError,
|
|
26
|
+
)
|
|
27
|
+
from carvustrain.exporter import ModelExporter
|
|
28
|
+
from carvustrain.logger import logger
|
|
29
|
+
from carvustrain.model import (
|
|
30
|
+
AgentModel,
|
|
31
|
+
ChatModel,
|
|
32
|
+
CustomModel,
|
|
33
|
+
LanguageModel,
|
|
34
|
+
Model,
|
|
35
|
+
TextModel,
|
|
36
|
+
)
|
|
37
|
+
from carvustrain.optimizer import SGD, AdaGrad, Adam, AdamW, Optimizer, RMSprop
|
|
38
|
+
from carvustrain.parser import CarvusTrainParser, DataParser
|
|
39
|
+
from carvustrain.scheduler import (
|
|
40
|
+
CosineAnnealingLR,
|
|
41
|
+
ExponentialLR,
|
|
42
|
+
LinearWarmupLR,
|
|
43
|
+
ReduceLROnPlateau,
|
|
44
|
+
StepLR,
|
|
45
|
+
)
|
|
46
|
+
from carvustrain.tokenizer import (
|
|
47
|
+
BPETokenizer,
|
|
48
|
+
CharTokenizer,
|
|
49
|
+
CustomTokenizer,
|
|
50
|
+
SentencePieceTokenizer,
|
|
51
|
+
SentenceTokenizer,
|
|
52
|
+
Tokenizer,
|
|
53
|
+
Vocabulary,
|
|
54
|
+
WordPieceTokenizer,
|
|
55
|
+
WordTokenizer,
|
|
56
|
+
)
|
|
57
|
+
from carvustrain.trainer import Trainer
|
|
58
|
+
from carvustrain.version import __version__
|
|
59
|
+
|
|
60
|
+
# Global default model instance for functional top-level API calls
|
|
61
|
+
_default_model: Optional[Model] = None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _get_default_model() -> Model:
|
|
65
|
+
global _default_model
|
|
66
|
+
if _default_model is None:
|
|
67
|
+
_default_model = Model(name="CarvusGlobal")
|
|
68
|
+
return _default_model
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def train(data: Optional[Union[str, Dataset, List[str]]] = None, **kwargs: Any) -> Dict[str, Any]:
|
|
72
|
+
"""Train global default model instance."""
|
|
73
|
+
return _get_default_model().train(data=data, **kwargs)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def ask(question: str) -> str:
|
|
77
|
+
"""Ask question to global default model instance."""
|
|
78
|
+
return _get_default_model().ask(question)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def predict(input_text: str) -> str:
|
|
82
|
+
"""Generate prediction using global default model instance."""
|
|
83
|
+
return _get_default_model().predict(input_text)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def generate(prompt: str, **kwargs: Any) -> str:
|
|
87
|
+
"""Generate text using global default model instance."""
|
|
88
|
+
return _get_default_model().generate(prompt, **kwargs)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def chat(user_message: str) -> str:
|
|
92
|
+
"""Chat with global default model instance."""
|
|
93
|
+
return _get_default_model().chat(user_message)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def learn(text: Union[str, List[str]]) -> None:
|
|
97
|
+
"""Ingest knowledge into global default model instance."""
|
|
98
|
+
_get_default_model().learn(text)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def finetune(data: Union[str, Dataset, List[str]], **kwargs: Any) -> Dict[str, Any]:
|
|
102
|
+
"""Fine-tune global default model instance."""
|
|
103
|
+
return _get_default_model().finetune(data=data, **kwargs)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def evaluate(dataset: Optional[Dataset] = None) -> Dict[str, float]:
|
|
107
|
+
"""Evaluate global default model instance."""
|
|
108
|
+
return _get_default_model().evaluate(dataset=dataset)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def export(output_path: str, format: Optional[str] = None) -> str:
|
|
112
|
+
"""Export global default model instance."""
|
|
113
|
+
return _get_default_model().export(output_path=output_path, format=format)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def save(filepath: Optional[str] = None) -> str:
|
|
117
|
+
"""Save global default model instance."""
|
|
118
|
+
return _get_default_model().save(filepath=filepath)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def load(filepath: str) -> Model:
|
|
122
|
+
"""Load model state from filepath."""
|
|
123
|
+
return _get_default_model().load(filepath=filepath)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def summary() -> None:
|
|
127
|
+
"""Display summary of global default model instance."""
|
|
128
|
+
_get_default_model().summary()
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def statistics() -> Dict[str, Any]:
|
|
132
|
+
"""Get statistics of global default model instance."""
|
|
133
|
+
return _get_default_model().statistics()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def memory() -> Dict[str, Any]:
|
|
137
|
+
"""Inspect memory of global default model instance."""
|
|
138
|
+
return _get_default_model().memory()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
__all__ = [
|
|
142
|
+
# Main Classes
|
|
143
|
+
"Model",
|
|
144
|
+
"ChatModel",
|
|
145
|
+
"TextModel",
|
|
146
|
+
"LanguageModel",
|
|
147
|
+
"CustomModel",
|
|
148
|
+
# Data & Tokenization
|
|
149
|
+
"Dataset",
|
|
150
|
+
"TextDataset",
|
|
151
|
+
"TabularDataset",
|
|
152
|
+
"DataLoader",
|
|
153
|
+
"Tokenizer",
|
|
154
|
+
"Vocabulary",
|
|
155
|
+
"WordTokenizer",
|
|
156
|
+
"CharTokenizer",
|
|
157
|
+
"SentenceTokenizer",
|
|
158
|
+
"BPETokenizer",
|
|
159
|
+
"WordPieceTokenizer",
|
|
160
|
+
"SentencePieceTokenizer",
|
|
161
|
+
"CustomTokenizer",
|
|
162
|
+
"DataParser",
|
|
163
|
+
"CarvusTrainParser",
|
|
164
|
+
# Engine & Optimization
|
|
165
|
+
"Trainer",
|
|
166
|
+
"Optimizer",
|
|
167
|
+
"Adam",
|
|
168
|
+
"AdamW",
|
|
169
|
+
"SGD",
|
|
170
|
+
"RMSprop",
|
|
171
|
+
"AdaGrad",
|
|
172
|
+
"StepLR",
|
|
173
|
+
"CosineAnnealingLR",
|
|
174
|
+
"LinearWarmupLR",
|
|
175
|
+
"ExponentialLR",
|
|
176
|
+
"ReduceLROnPlateau",
|
|
177
|
+
"ModelExporter",
|
|
178
|
+
"Evaluator",
|
|
179
|
+
"Benchmarker",
|
|
180
|
+
# Configuration & Logging
|
|
181
|
+
"CarvusConfig",
|
|
182
|
+
"ModelConfig",
|
|
183
|
+
"TrainingConfig",
|
|
184
|
+
"DatasetConfig",
|
|
185
|
+
"InferenceConfig",
|
|
186
|
+
"load_config_file",
|
|
187
|
+
"logger",
|
|
188
|
+
"__version__",
|
|
189
|
+
# Exceptions
|
|
190
|
+
"CarvusTrainError",
|
|
191
|
+
"DatasetNotFoundError",
|
|
192
|
+
"TrainingError",
|
|
193
|
+
"ModelError",
|
|
194
|
+
"ParserError",
|
|
195
|
+
"TokenizerError",
|
|
196
|
+
"ExportError",
|
|
197
|
+
"InferenceError",
|
|
198
|
+
"ConfigurationError",
|
|
199
|
+
# Modules
|
|
200
|
+
"activation",
|
|
201
|
+
"callbacks",
|
|
202
|
+
"layers",
|
|
203
|
+
"losses",
|
|
204
|
+
"metrics",
|
|
205
|
+
# Functional APIs
|
|
206
|
+
"train",
|
|
207
|
+
"ask",
|
|
208
|
+
"predict",
|
|
209
|
+
"generate",
|
|
210
|
+
"chat",
|
|
211
|
+
"learn",
|
|
212
|
+
"finetune",
|
|
213
|
+
"evaluate",
|
|
214
|
+
"export",
|
|
215
|
+
"save",
|
|
216
|
+
"load",
|
|
217
|
+
"summary",
|
|
218
|
+
"statistics",
|
|
219
|
+
"memory",
|
|
220
|
+
]
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"""Activation functions for neural networks with NumPy and PyTorch support."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from typing import Any, List, Union
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Activation:
|
|
8
|
+
"""Base activation class."""
|
|
9
|
+
|
|
10
|
+
def __call__(self, x: Any) -> Any:
|
|
11
|
+
return self.forward(x)
|
|
12
|
+
|
|
13
|
+
def forward(self, x: Any) -> Any:
|
|
14
|
+
raise NotImplementedError
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ReLU(Activation):
|
|
18
|
+
"""Rectified Linear Unit activation."""
|
|
19
|
+
|
|
20
|
+
def forward(self, x: Any) -> Any:
|
|
21
|
+
try:
|
|
22
|
+
import torch
|
|
23
|
+
|
|
24
|
+
if isinstance(x, torch.Tensor):
|
|
25
|
+
return torch.relu(x)
|
|
26
|
+
except ImportError:
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
import numpy as np
|
|
31
|
+
|
|
32
|
+
if isinstance(x, np.ndarray):
|
|
33
|
+
return np.maximum(0, x)
|
|
34
|
+
except ImportError:
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
if isinstance(x, (list, tuple)):
|
|
38
|
+
return [max(0.0, float(v)) for v in x]
|
|
39
|
+
return max(0.0, float(x))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class GELU(Activation):
|
|
43
|
+
"""Gaussian Error Linear Unit activation."""
|
|
44
|
+
|
|
45
|
+
def forward(self, x: Any) -> Any:
|
|
46
|
+
try:
|
|
47
|
+
import torch
|
|
48
|
+
|
|
49
|
+
if isinstance(x, torch.Tensor):
|
|
50
|
+
return torch.nn.functional.gelu(x)
|
|
51
|
+
except ImportError:
|
|
52
|
+
pass
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
import numpy as np
|
|
56
|
+
|
|
57
|
+
if isinstance(x, np.ndarray):
|
|
58
|
+
return 0.5 * x * (1.0 + np.tanh(np.sqrt(2.0 / np.pi) * (x + 0.044715 * np.power(x, 3))))
|
|
59
|
+
except ImportError:
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
# Scalar fallback
|
|
63
|
+
v = float(x)
|
|
64
|
+
return 0.5 * v * (1.0 + math.tanh(math.sqrt(2.0 / math.pi) * (v + 0.044715 * (v**3))))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class SiLU(Activation):
|
|
68
|
+
"""Sigmoid Linear Unit (Swish) activation."""
|
|
69
|
+
|
|
70
|
+
def forward(self, x: Any) -> Any:
|
|
71
|
+
try:
|
|
72
|
+
import torch
|
|
73
|
+
|
|
74
|
+
if isinstance(x, torch.Tensor):
|
|
75
|
+
return torch.nn.functional.silu(x)
|
|
76
|
+
except ImportError:
|
|
77
|
+
pass
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
import numpy as np
|
|
81
|
+
|
|
82
|
+
if isinstance(x, np.ndarray):
|
|
83
|
+
return x / (1.0 + np.exp(-x))
|
|
84
|
+
except ImportError:
|
|
85
|
+
pass
|
|
86
|
+
|
|
87
|
+
v = float(x)
|
|
88
|
+
return v / (1.0 + math.exp(-v))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class Softmax(Activation):
|
|
92
|
+
"""Softmax activation along specified dimension."""
|
|
93
|
+
|
|
94
|
+
def __init__(self, dim: int = -1) -> None:
|
|
95
|
+
self.dim = dim
|
|
96
|
+
|
|
97
|
+
def forward(self, x: Any) -> Any:
|
|
98
|
+
try:
|
|
99
|
+
import torch
|
|
100
|
+
|
|
101
|
+
if isinstance(x, torch.Tensor):
|
|
102
|
+
return torch.softmax(x, dim=self.dim)
|
|
103
|
+
except ImportError:
|
|
104
|
+
pass
|
|
105
|
+
|
|
106
|
+
try:
|
|
107
|
+
import numpy as np
|
|
108
|
+
|
|
109
|
+
if isinstance(x, np.ndarray):
|
|
110
|
+
exp_x = np.exp(x - np.max(x, axis=self.dim, keepdims=True))
|
|
111
|
+
return exp_x / np.sum(exp_x, axis=self.dim, keepdims=True)
|
|
112
|
+
except ImportError:
|
|
113
|
+
pass
|
|
114
|
+
|
|
115
|
+
if isinstance(x, (list, tuple)):
|
|
116
|
+
max_v = max(x)
|
|
117
|
+
exps = [math.exp(v - max_v) for v in x]
|
|
118
|
+
sum_exps = sum(exps)
|
|
119
|
+
return [e / sum_exps for e in exps]
|
|
120
|
+
return 1.0
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class Sigmoid(Activation):
|
|
124
|
+
"""Sigmoid activation."""
|
|
125
|
+
|
|
126
|
+
def forward(self, x: Any) -> Any:
|
|
127
|
+
try:
|
|
128
|
+
import torch
|
|
129
|
+
|
|
130
|
+
if isinstance(x, torch.Tensor):
|
|
131
|
+
return torch.sigmoid(x)
|
|
132
|
+
except ImportError:
|
|
133
|
+
pass
|
|
134
|
+
|
|
135
|
+
try:
|
|
136
|
+
import numpy as np
|
|
137
|
+
|
|
138
|
+
if isinstance(x, np.ndarray):
|
|
139
|
+
return 1.0 / (1.0 + np.exp(-x))
|
|
140
|
+
except ImportError:
|
|
141
|
+
pass
|
|
142
|
+
|
|
143
|
+
return 1.0 / (1.0 + math.exp(-float(x)))
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class Tanh(Activation):
|
|
147
|
+
"""Hyperbolic tangent activation."""
|
|
148
|
+
|
|
149
|
+
def forward(self, x: Any) -> Any:
|
|
150
|
+
try:
|
|
151
|
+
import torch
|
|
152
|
+
|
|
153
|
+
if isinstance(x, torch.Tensor):
|
|
154
|
+
return torch.tanh(x)
|
|
155
|
+
except ImportError:
|
|
156
|
+
pass
|
|
157
|
+
|
|
158
|
+
try:
|
|
159
|
+
import numpy as np
|
|
160
|
+
|
|
161
|
+
if isinstance(x, np.ndarray):
|
|
162
|
+
return np.tanh(x)
|
|
163
|
+
except ImportError:
|
|
164
|
+
pass
|
|
165
|
+
|
|
166
|
+
return math.tanh(float(x))
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class LeakyReLU(Activation):
|
|
170
|
+
"""Leaky Rectified Linear Unit activation."""
|
|
171
|
+
|
|
172
|
+
def __init__(self, negative_slope: float = 0.01) -> None:
|
|
173
|
+
self.negative_slope = negative_slope
|
|
174
|
+
|
|
175
|
+
def forward(self, x: Any) -> Any:
|
|
176
|
+
try:
|
|
177
|
+
import torch
|
|
178
|
+
|
|
179
|
+
if isinstance(x, torch.Tensor):
|
|
180
|
+
return torch.nn.functional.leaky_relu(x, negative_slope=self.negative_slope)
|
|
181
|
+
except ImportError:
|
|
182
|
+
pass
|
|
183
|
+
|
|
184
|
+
try:
|
|
185
|
+
import numpy as np
|
|
186
|
+
|
|
187
|
+
if isinstance(x, np.ndarray):
|
|
188
|
+
return np.where(x > 0, x, x * self.negative_slope)
|
|
189
|
+
except ImportError:
|
|
190
|
+
pass
|
|
191
|
+
|
|
192
|
+
v = float(x)
|
|
193
|
+
return v if v > 0 else v * self.negative_slope
|
CarvusTrain/callbacks.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Callback infrastructure for monitoring, early stopping, checkpointing, and logging during training."""
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
from typing import Any, Dict, List, Optional
|
|
7
|
+
|
|
8
|
+
from carvustrain.logger import logger
|
|
9
|
+
from carvustrain.utils import format_bytes, format_time
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Callback:
|
|
13
|
+
"""Base callback class for training hooks."""
|
|
14
|
+
|
|
15
|
+
def on_train_begin(self, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
def on_train_end(self, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
def on_epoch_begin(self, epoch: int, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
def on_epoch_end(self, epoch: int, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
def on_batch_begin(self, batch: int, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
def on_batch_end(self, batch: int, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class EarlyStopping(Callback):
|
|
35
|
+
"""Stop training when a monitored metric stops improving."""
|
|
36
|
+
|
|
37
|
+
def __init__(self, monitor: str = "val_loss", patience: int = 5, min_delta: float = 1e-4, mode: str = "min") -> None:
|
|
38
|
+
self.monitor = monitor
|
|
39
|
+
self.patience = patience
|
|
40
|
+
self.min_delta = min_delta
|
|
41
|
+
self.mode = mode
|
|
42
|
+
self.best_val = float("inf") if mode == "min" else float("-inf")
|
|
43
|
+
self.wait = 0
|
|
44
|
+
self.stopped_epoch = 0
|
|
45
|
+
self.should_stop = False
|
|
46
|
+
|
|
47
|
+
def on_epoch_end(self, epoch: int, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
48
|
+
logs = logs or {}
|
|
49
|
+
current = logs.get(self.monitor)
|
|
50
|
+
if current is None:
|
|
51
|
+
return
|
|
52
|
+
|
|
53
|
+
is_better = (current < self.best_val - self.min_delta) if self.mode == "min" else (current > self.best_val + self.min_delta)
|
|
54
|
+
if is_better:
|
|
55
|
+
self.best_val = current
|
|
56
|
+
self.wait = 0
|
|
57
|
+
else:
|
|
58
|
+
self.wait += 1
|
|
59
|
+
if self.wait >= self.patience:
|
|
60
|
+
self.stopped_epoch = epoch
|
|
61
|
+
self.should_stop = True
|
|
62
|
+
logger.warning(f"Early stopping triggered at epoch {epoch} (best {self.monitor}: {self.best_val:.4f})")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class ModelCheckpoint(Callback):
|
|
66
|
+
"""Save model checkpoint file at regular intervals or when metric improves."""
|
|
67
|
+
|
|
68
|
+
def __init__(self, filepath: str = "checkpoints/checkpoint_epoch_{epoch}.ct", save_best_only: bool = True, monitor: str = "val_loss") -> None:
|
|
69
|
+
self.filepath = filepath
|
|
70
|
+
self.save_best_only = save_best_only
|
|
71
|
+
self.monitor = monitor
|
|
72
|
+
self.best_val = float("inf")
|
|
73
|
+
|
|
74
|
+
def on_epoch_end(self, epoch: int, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
75
|
+
logs = logs or {}
|
|
76
|
+
model = logs.get("model")
|
|
77
|
+
current = logs.get(self.monitor, logs.get("loss"))
|
|
78
|
+
|
|
79
|
+
if model is None:
|
|
80
|
+
return
|
|
81
|
+
|
|
82
|
+
if self.save_best_only and current is not None:
|
|
83
|
+
if current < self.best_val:
|
|
84
|
+
self.best_val = current
|
|
85
|
+
save_path = self.filepath.format(epoch=epoch)
|
|
86
|
+
model.save(save_path)
|
|
87
|
+
logger.info(f"Checkpoint saved to '{save_path}' ({self.monitor}: {current:.4f})")
|
|
88
|
+
else:
|
|
89
|
+
save_path = self.filepath.format(epoch=epoch)
|
|
90
|
+
model.save(save_path)
|
|
91
|
+
logger.info(f"Checkpoint saved to '{save_path}'")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class ProgressBarCallback(Callback):
|
|
95
|
+
"""Console progress logging for training iterations."""
|
|
96
|
+
|
|
97
|
+
def __init__(self) -> None:
|
|
98
|
+
self.start_time: float = 0.0
|
|
99
|
+
|
|
100
|
+
def on_epoch_begin(self, epoch: int, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
101
|
+
self.start_time = time.time()
|
|
102
|
+
|
|
103
|
+
def on_epoch_end(self, epoch: int, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
104
|
+
logs = logs or {}
|
|
105
|
+
elapsed = time.time() - self.start_time
|
|
106
|
+
total_epochs = logs.get("total_epochs", 0)
|
|
107
|
+
loss = logs.get("loss", 0.0)
|
|
108
|
+
acc = logs.get("accuracy")
|
|
109
|
+
lr = logs.get("lr")
|
|
110
|
+
logger.log_epoch(epoch, total_epochs, loss, accuracy=acc, lr=lr, elapsed=elapsed)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class CSVLoggerCallback(Callback):
|
|
114
|
+
"""Log training progress to a CSV spreadsheet file."""
|
|
115
|
+
|
|
116
|
+
def __init__(self, filename: str = "training_log.csv") -> None:
|
|
117
|
+
self.filename = filename
|
|
118
|
+
self.file_opened = False
|
|
119
|
+
|
|
120
|
+
def on_epoch_end(self, epoch: int, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
121
|
+
logs = logs or {}
|
|
122
|
+
row = {"epoch": epoch, "loss": logs.get("loss"), "accuracy": logs.get("accuracy"), "lr": logs.get("lr")}
|
|
123
|
+
|
|
124
|
+
os.makedirs(os.path.dirname(os.path.abspath(self.filename)), exist_ok=True)
|
|
125
|
+
file_exists = os.path.exists(self.filename)
|
|
126
|
+
|
|
127
|
+
with open(self.filename, "a", newline="", encoding="utf-8") as f:
|
|
128
|
+
writer = csv.DictWriter(f, fieldnames=list(row.keys()))
|
|
129
|
+
if not file_exists:
|
|
130
|
+
writer.writeheader()
|
|
131
|
+
writer.writerow(row)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class MemoryTrackerCallback(Callback):
|
|
135
|
+
"""Monitors CPU/GPU memory footprint during training loop."""
|
|
136
|
+
|
|
137
|
+
def on_epoch_end(self, epoch: int, logs: Optional[Dict[str, Any]] = None) -> None:
|
|
138
|
+
try:
|
|
139
|
+
import torch
|
|
140
|
+
|
|
141
|
+
if torch.cuda.is_available():
|
|
142
|
+
allocated = torch.cuda.memory_allocated(0)
|
|
143
|
+
reserved = torch.cuda.memory_reserved(0)
|
|
144
|
+
logger.debug(f"[Memory] GPU Allocated: {format_bytes(allocated)} | Reserved: {format_bytes(reserved)}")
|
|
145
|
+
except ImportError:
|
|
146
|
+
pass
|