retrainopt 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- retrainopt/__init__.py +31 -0
- retrainopt/__main__.py +8 -0
- retrainopt/adapters/__init__.py +13 -0
- retrainopt/adapters/model.py +216 -0
- retrainopt/adapters/timeline.py +166 -0
- retrainopt/cli.py +130 -0
- retrainopt/config.py +106 -0
- retrainopt/core/__init__.py +20 -0
- retrainopt/core/acquisition.py +106 -0
- retrainopt/core/bayes_backends.py +282 -0
- retrainopt/core/costs.py +14 -0
- retrainopt/core/dp.py +208 -0
- retrainopt/core/surrogate.py +273 -0
- retrainopt/evaluation.py +170 -0
- retrainopt/interfaces.py +79 -0
- retrainopt/planner.py +677 -0
- retrainopt/py.typed +0 -0
- retrainopt/types.py +102 -0
- retrainopt-0.2.0.dist-info/METADATA +134 -0
- retrainopt-0.2.0.dist-info/RECORD +24 -0
- retrainopt-0.2.0.dist-info/WHEEL +5 -0
- retrainopt-0.2.0.dist-info/entry_points.txt +2 -0
- retrainopt-0.2.0.dist-info/licenses/LICENSE +184 -0
- retrainopt-0.2.0.dist-info/top_level.txt +1 -0
retrainopt/__init__.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""RetrainOpt public package.
|
|
2
|
+
|
|
3
|
+
RetrainOpt provides cost-aware retraining planning for time-sliced datasets.
|
|
4
|
+
The public API is intentionally small:
|
|
5
|
+
|
|
6
|
+
- ``TimelineDataset`` describes the retraining timeline.
|
|
7
|
+
- ``ModelAdapter`` abstracts model fitting and evaluation.
|
|
8
|
+
- ``PlannerConfig`` controls the search space and planning objective.
|
|
9
|
+
- ``RetrainingPlanner`` produces retraining decision sequences.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from importlib.metadata import version
|
|
13
|
+
|
|
14
|
+
from .config import PlannerConfig
|
|
15
|
+
from .interfaces import EvaluationProvider, ModelAdapter, TimelineDataset
|
|
16
|
+
from .planner import RetrainingPlanner
|
|
17
|
+
from .types import CandidateEvaluation, Decision, DecisionAction, PlanResult
|
|
18
|
+
|
|
19
|
+
__version__ = version("retrainopt")
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"Decision",
|
|
23
|
+
"CandidateEvaluation",
|
|
24
|
+
"DecisionAction",
|
|
25
|
+
"EvaluationProvider",
|
|
26
|
+
"ModelAdapter",
|
|
27
|
+
"PlannerConfig",
|
|
28
|
+
"PlanResult",
|
|
29
|
+
"RetrainingPlanner",
|
|
30
|
+
"TimelineDataset",
|
|
31
|
+
]
|
retrainopt/__main__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Built-in timeline and model adapters."""
|
|
2
|
+
|
|
3
|
+
from .model import PrototypeModelAdapter, SklearnModelAdapter, TorchModelAdapter
|
|
4
|
+
from .timeline import ArrayTimelineAdapter, BinaryTimelineAdapter, TorchTimelineAdapter
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
"ArrayTimelineAdapter",
|
|
8
|
+
"BinaryTimelineAdapter",
|
|
9
|
+
"PrototypeModelAdapter",
|
|
10
|
+
"SklearnModelAdapter",
|
|
11
|
+
"TorchModelAdapter",
|
|
12
|
+
"TorchTimelineAdapter",
|
|
13
|
+
]
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Built-in model adapters."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from numbers import Integral
|
|
6
|
+
from typing import Any, Callable, Iterable, List, Sequence, Tuple
|
|
7
|
+
|
|
8
|
+
from ..interfaces import ModelAdapter
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _split_labeled_block(block: Any) -> Tuple[List[Any], List[int]]:
|
|
12
|
+
features: List[Any] = []
|
|
13
|
+
labels: List[int] = []
|
|
14
|
+
for item in block:
|
|
15
|
+
if not isinstance(item, Sequence) or len(item) != 2:
|
|
16
|
+
raise ValueError("Expected labeled items shaped like (features, label).")
|
|
17
|
+
features.append(item[0])
|
|
18
|
+
labels.append(int(item[1]))
|
|
19
|
+
return features, labels
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SklearnModelAdapter(ModelAdapter):
|
|
23
|
+
"""ModelAdapter for sklearn classifiers."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, model_factory: Callable[[], Any] | Any):
|
|
26
|
+
self.model_factory = model_factory
|
|
27
|
+
|
|
28
|
+
def fit(self, train_data: Any, *, step: int, n_samples: int, random_seed: int) -> Any:
|
|
29
|
+
estimator = self._build_estimator()
|
|
30
|
+
if hasattr(estimator, "get_params"):
|
|
31
|
+
seed_params = {
|
|
32
|
+
name: int(random_seed) % (2 ** 32)
|
|
33
|
+
for name, value in estimator.get_params(deep=True).items()
|
|
34
|
+
if (name == "random_state" or name.endswith("__random_state")) and value is None
|
|
35
|
+
}
|
|
36
|
+
if seed_params:
|
|
37
|
+
estimator.set_params(**seed_params)
|
|
38
|
+
features, labels = _split_labeled_block(train_data)
|
|
39
|
+
estimator.fit(features, labels)
|
|
40
|
+
return estimator
|
|
41
|
+
|
|
42
|
+
def evaluate(self, model: Any, eval_data: Any, *, step: int) -> float:
|
|
43
|
+
try:
|
|
44
|
+
from sklearn.metrics import log_loss
|
|
45
|
+
except ImportError as exc: # pragma: no cover - optional dependency
|
|
46
|
+
raise RuntimeError("SklearnModelAdapter requires the `sklearn` extra.") from exc
|
|
47
|
+
|
|
48
|
+
features, labels = _split_labeled_block(eval_data)
|
|
49
|
+
if not features:
|
|
50
|
+
raise ValueError("Evaluation data must not be empty.")
|
|
51
|
+
if hasattr(model, "predict_proba"):
|
|
52
|
+
probabilities = model.predict_proba(features)
|
|
53
|
+
return float(log_loss(labels, probabilities, labels=list(model.classes_)))
|
|
54
|
+
predictions = model.predict(features)
|
|
55
|
+
mistakes = sum(int(prediction != label) for prediction, label in zip(predictions, labels))
|
|
56
|
+
return mistakes / max(1, len(labels))
|
|
57
|
+
|
|
58
|
+
def _build_estimator(self) -> Any:
|
|
59
|
+
try:
|
|
60
|
+
from sklearn.base import clone
|
|
61
|
+
except ImportError as exc: # pragma: no cover - optional dependency
|
|
62
|
+
raise RuntimeError("SklearnModelAdapter requires the `sklearn` extra.") from exc
|
|
63
|
+
|
|
64
|
+
if callable(self.model_factory):
|
|
65
|
+
return self.model_factory()
|
|
66
|
+
return clone(self.model_factory)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class PrototypeModelAdapter(ModelAdapter):
|
|
70
|
+
"""Dependency-free prototype classifier for small structured datasets."""
|
|
71
|
+
|
|
72
|
+
def fit(self, train_data: Any, *, step: int, n_samples: int, random_seed: int) -> Any:
|
|
73
|
+
if not train_data:
|
|
74
|
+
return {"prototypes": {}, "majority": 0}
|
|
75
|
+
|
|
76
|
+
grouped = {}
|
|
77
|
+
counts = {}
|
|
78
|
+
for features, label in train_data:
|
|
79
|
+
feature_vector = self._flatten_features(features)
|
|
80
|
+
grouped.setdefault(int(label), []).append(feature_vector)
|
|
81
|
+
counts[int(label)] = counts.get(int(label), 0) + 1
|
|
82
|
+
|
|
83
|
+
prototypes = {}
|
|
84
|
+
for label, vectors in grouped.items():
|
|
85
|
+
dimension = len(vectors[0])
|
|
86
|
+
mean_vector = []
|
|
87
|
+
for index in range(dimension):
|
|
88
|
+
mean_vector.append(sum(vector[index] for vector in vectors) / len(vectors))
|
|
89
|
+
prototypes[label] = tuple(mean_vector)
|
|
90
|
+
|
|
91
|
+
majority = max(counts.items(), key=lambda item: item[1])[0]
|
|
92
|
+
return {"prototypes": prototypes, "majority": majority}
|
|
93
|
+
|
|
94
|
+
def evaluate(self, model: Any, eval_data: Any, *, step: int) -> float:
|
|
95
|
+
if not eval_data:
|
|
96
|
+
raise ValueError("Evaluation data must not be empty.")
|
|
97
|
+
|
|
98
|
+
mistakes = 0
|
|
99
|
+
prototypes = model.get("prototypes", {})
|
|
100
|
+
majority = model.get("majority", 0)
|
|
101
|
+
for features, label in eval_data:
|
|
102
|
+
feature_vector = self._flatten_features(features)
|
|
103
|
+
if prototypes:
|
|
104
|
+
predicted = min(
|
|
105
|
+
prototypes.items(),
|
|
106
|
+
key=lambda item: self._squared_distance(item[1], feature_vector),
|
|
107
|
+
)[0]
|
|
108
|
+
else:
|
|
109
|
+
predicted = majority
|
|
110
|
+
if int(predicted) != int(label):
|
|
111
|
+
mistakes += 1
|
|
112
|
+
return mistakes / len(eval_data)
|
|
113
|
+
|
|
114
|
+
@staticmethod
|
|
115
|
+
def _flatten_features(features: Any) -> Tuple[float, ...]:
|
|
116
|
+
if isinstance(features, (list, tuple)):
|
|
117
|
+
return tuple(float(value) for value in features)
|
|
118
|
+
return (float(features),)
|
|
119
|
+
|
|
120
|
+
@staticmethod
|
|
121
|
+
def _squared_distance(left: Sequence[float], right: Sequence[float]) -> float:
|
|
122
|
+
return sum((float(lv) - float(rv)) ** 2 for lv, rv in zip(left, right))
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class TorchModelAdapter(ModelAdapter):
|
|
126
|
+
"""ModelAdapter for PyTorch classifiers."""
|
|
127
|
+
|
|
128
|
+
def __init__(
|
|
129
|
+
self,
|
|
130
|
+
model_factory: Callable[[], Any],
|
|
131
|
+
*,
|
|
132
|
+
optimizer_factory: Callable[[Iterable[Any]], Any] | None = None,
|
|
133
|
+
epochs: int = 5,
|
|
134
|
+
batch_size: int = 64,
|
|
135
|
+
device: str = "cpu",
|
|
136
|
+
):
|
|
137
|
+
self.model_factory = model_factory
|
|
138
|
+
self.optimizer_factory = optimizer_factory
|
|
139
|
+
for name, value in (("epochs", epochs), ("batch_size", batch_size)):
|
|
140
|
+
if isinstance(value, bool) or not isinstance(value, Integral) or value < 1:
|
|
141
|
+
raise ValueError(f"{name} must be a positive integer.")
|
|
142
|
+
self.epochs = int(epochs)
|
|
143
|
+
self.batch_size = int(batch_size)
|
|
144
|
+
self.device = device
|
|
145
|
+
|
|
146
|
+
def fit(self, train_data: Any, *, step: int, n_samples: int, random_seed: int) -> Any:
|
|
147
|
+
try:
|
|
148
|
+
import torch
|
|
149
|
+
from torch.utils.data import DataLoader
|
|
150
|
+
except ImportError as exc: # pragma: no cover - optional dependency
|
|
151
|
+
raise RuntimeError("TorchModelAdapter requires the `torch` extra.") from exc
|
|
152
|
+
|
|
153
|
+
device = torch.device(self.device)
|
|
154
|
+
device_type = device.type if device.type in {"cuda", "mps"} else "cuda"
|
|
155
|
+
devices = []
|
|
156
|
+
if device.type in {"cuda", "mps"}:
|
|
157
|
+
devices = [device.index or 0]
|
|
158
|
+
with torch.random.fork_rng(devices=devices, device_type=device_type):
|
|
159
|
+
torch.random.default_generator.manual_seed(random_seed)
|
|
160
|
+
if device.type == "cuda":
|
|
161
|
+
with torch.cuda.device(devices[0]):
|
|
162
|
+
torch.cuda.manual_seed(random_seed)
|
|
163
|
+
elif device.type == "mps":
|
|
164
|
+
torch.mps.manual_seed(random_seed)
|
|
165
|
+
model = self.model_factory().to(self.device)
|
|
166
|
+
optimizer = (
|
|
167
|
+
self.optimizer_factory(model.parameters())
|
|
168
|
+
if self.optimizer_factory is not None
|
|
169
|
+
else torch.optim.Adam(model.parameters(), lr=1e-3)
|
|
170
|
+
)
|
|
171
|
+
criterion = torch.nn.CrossEntropyLoss()
|
|
172
|
+
generator = torch.Generator().manual_seed(random_seed)
|
|
173
|
+
train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True, generator=generator)
|
|
174
|
+
|
|
175
|
+
model.train()
|
|
176
|
+
for _ in range(self.epochs):
|
|
177
|
+
for inputs, targets in train_loader:
|
|
178
|
+
inputs = inputs.to(self.device)
|
|
179
|
+
targets = targets.to(self.device)
|
|
180
|
+
optimizer.zero_grad()
|
|
181
|
+
outputs = model(inputs)
|
|
182
|
+
loss = criterion(outputs, targets)
|
|
183
|
+
loss.backward()
|
|
184
|
+
optimizer.step()
|
|
185
|
+
if device.type == "cuda":
|
|
186
|
+
torch.cuda.synchronize(device)
|
|
187
|
+
elif device.type == "mps":
|
|
188
|
+
torch.mps.synchronize()
|
|
189
|
+
|
|
190
|
+
return model
|
|
191
|
+
|
|
192
|
+
def evaluate(self, model: Any, eval_data: Any, *, step: int) -> float:
|
|
193
|
+
try:
|
|
194
|
+
import torch
|
|
195
|
+
from torch.utils.data import DataLoader
|
|
196
|
+
except ImportError as exc: # pragma: no cover - optional dependency
|
|
197
|
+
raise RuntimeError("TorchModelAdapter requires the `torch` extra.") from exc
|
|
198
|
+
|
|
199
|
+
loader = DataLoader(eval_data, batch_size=self.batch_size, shuffle=False, generator=torch.Generator().manual_seed(0))
|
|
200
|
+
criterion = torch.nn.CrossEntropyLoss()
|
|
201
|
+
model.eval()
|
|
202
|
+
total_loss = 0.0
|
|
203
|
+
total_count = 0
|
|
204
|
+
|
|
205
|
+
with torch.no_grad():
|
|
206
|
+
for inputs, targets in loader:
|
|
207
|
+
inputs = inputs.to(self.device)
|
|
208
|
+
targets = targets.to(self.device)
|
|
209
|
+
outputs = model(inputs)
|
|
210
|
+
loss = criterion(outputs, targets)
|
|
211
|
+
total_loss += float(loss.item()) * int(inputs.size(0))
|
|
212
|
+
total_count += int(inputs.size(0))
|
|
213
|
+
|
|
214
|
+
if total_count <= 0:
|
|
215
|
+
raise ValueError("Evaluation data must not be empty.")
|
|
216
|
+
return total_loss / total_count
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""Built-in timeline adapters."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
import random
|
|
8
|
+
import struct
|
|
9
|
+
from numbers import Integral
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Iterable, List, Sequence
|
|
12
|
+
|
|
13
|
+
from ..interfaces import TimelineDataset
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ArrayTimelineAdapter(TimelineDataset):
|
|
17
|
+
"""Timeline adapter backed by in-memory Python sequences."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, train_blocks: Sequence[Sequence[Any]], eval_blocks: Sequence[Sequence[Any]]):
|
|
20
|
+
if len(train_blocks) != len(eval_blocks):
|
|
21
|
+
raise ValueError("train_blocks and eval_blocks must have the same number of timesteps.")
|
|
22
|
+
if len(train_blocks) == 0:
|
|
23
|
+
raise ValueError("A timeline must contain at least one timestep.")
|
|
24
|
+
self._train_blocks = [list(block) for block in train_blocks]
|
|
25
|
+
self._eval_blocks = [list(block) for block in eval_blocks]
|
|
26
|
+
if any(not block for block in self._eval_blocks):
|
|
27
|
+
raise ValueError("Evaluation blocks must not be empty.")
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def num_steps(self) -> int:
|
|
31
|
+
return len(self._train_blocks)
|
|
32
|
+
|
|
33
|
+
def get_train_block(self, step: int) -> List[Any]:
|
|
34
|
+
return self._train_blocks[step]
|
|
35
|
+
|
|
36
|
+
def get_eval_block(self, step: int) -> List[Any]:
|
|
37
|
+
return self._eval_blocks[step]
|
|
38
|
+
|
|
39
|
+
def train_size(self, step: int) -> int:
|
|
40
|
+
return len(self._train_blocks[step])
|
|
41
|
+
|
|
42
|
+
def sample_train_block(self, step: int, n_samples: int, *, seed: int) -> List[Any]:
|
|
43
|
+
block = self._train_blocks[step]
|
|
44
|
+
actual_n = min(max(0, int(n_samples)), len(block))
|
|
45
|
+
if actual_n >= len(block):
|
|
46
|
+
return list(block)
|
|
47
|
+
if actual_n <= 0:
|
|
48
|
+
return []
|
|
49
|
+
rng = random.Random(seed)
|
|
50
|
+
indices = sorted(rng.sample(range(len(block)), actual_n))
|
|
51
|
+
return [block[idx] for idx in indices]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class TorchTimelineAdapter(TimelineDataset):
|
|
55
|
+
"""Timeline adapter for PyTorch datasets or lists of tensor/label pairs."""
|
|
56
|
+
|
|
57
|
+
def __init__(self, train_blocks: Sequence[Any], eval_blocks: Sequence[Any]):
|
|
58
|
+
if len(train_blocks) != len(eval_blocks):
|
|
59
|
+
raise ValueError("train_blocks and eval_blocks must have the same number of timesteps.")
|
|
60
|
+
if len(train_blocks) == 0:
|
|
61
|
+
raise ValueError("A timeline must contain at least one timestep.")
|
|
62
|
+
self._train_blocks = list(train_blocks)
|
|
63
|
+
self._eval_blocks = list(eval_blocks)
|
|
64
|
+
if any(len(block) == 0 for block in self._eval_blocks):
|
|
65
|
+
raise ValueError("Evaluation blocks must not be empty.")
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def num_steps(self) -> int:
|
|
69
|
+
return len(self._train_blocks)
|
|
70
|
+
|
|
71
|
+
def get_train_block(self, step: int) -> Any:
|
|
72
|
+
return self._train_blocks[step]
|
|
73
|
+
|
|
74
|
+
def get_eval_block(self, step: int) -> Any:
|
|
75
|
+
return self._eval_blocks[step]
|
|
76
|
+
|
|
77
|
+
def train_size(self, step: int) -> int:
|
|
78
|
+
return len(self._train_blocks[step])
|
|
79
|
+
|
|
80
|
+
def sample_train_block(self, step: int, n_samples: int, *, seed: int) -> Any:
|
|
81
|
+
block = self._train_blocks[step]
|
|
82
|
+
actual_n = min(max(0, int(n_samples)), len(block))
|
|
83
|
+
if actual_n >= len(block):
|
|
84
|
+
return block
|
|
85
|
+
if actual_n <= 0:
|
|
86
|
+
return []
|
|
87
|
+
|
|
88
|
+
try:
|
|
89
|
+
from torch.utils.data import Subset
|
|
90
|
+
except ImportError as exc: # pragma: no cover - optional dependency
|
|
91
|
+
raise RuntimeError("TorchTimelineAdapter requires the `torch` extra to sample datasets.") from exc
|
|
92
|
+
|
|
93
|
+
rng = random.Random(seed)
|
|
94
|
+
indices = sorted(rng.sample(range(len(block)), actual_n))
|
|
95
|
+
return Subset(block, indices)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class BinaryTimelineAdapter(ArrayTimelineAdapter):
|
|
99
|
+
"""Load a ``train/*.bin`` and ``test/*.bin`` timeline directory."""
|
|
100
|
+
|
|
101
|
+
def __init__(self, root_dir: str | Path, *, num_features: int | None = None):
|
|
102
|
+
root_path = Path(root_dir)
|
|
103
|
+
metadata_path = root_path / "metadata.json"
|
|
104
|
+
if num_features is None and metadata_path.exists():
|
|
105
|
+
metadata = json.loads(metadata_path.read_text())
|
|
106
|
+
if not isinstance(metadata, dict):
|
|
107
|
+
raise ValueError("metadata.json must contain a JSON object.")
|
|
108
|
+
num_features = metadata.get("num_features")
|
|
109
|
+
if isinstance(num_features, bool) or not isinstance(num_features, Integral) or num_features < 1:
|
|
110
|
+
raise ValueError("num_features must be a positive integer, supplied explicitly or in metadata.json.")
|
|
111
|
+
|
|
112
|
+
train_dir = root_path / "train"
|
|
113
|
+
eval_dir = root_path / "test"
|
|
114
|
+
train_ids = self._block_ids(train_dir)
|
|
115
|
+
eval_ids = self._block_ids(eval_dir)
|
|
116
|
+
if train_ids != eval_ids:
|
|
117
|
+
raise ValueError("Train and evaluation block identifiers must match exactly.")
|
|
118
|
+
train_blocks = self._load_blocks(train_dir, num_features)
|
|
119
|
+
eval_blocks = self._load_blocks(eval_dir, num_features)
|
|
120
|
+
super().__init__(train_blocks=train_blocks, eval_blocks=eval_blocks)
|
|
121
|
+
|
|
122
|
+
@staticmethod
|
|
123
|
+
def _block_ids(directory: Path) -> set[str]:
|
|
124
|
+
if not directory.is_dir():
|
|
125
|
+
raise FileNotFoundError(f"Timeline directory not found: {directory}")
|
|
126
|
+
stems = {path.stem for path in directory.glob("*.bin")}
|
|
127
|
+
numeric_ids = [int(stem) for stem in stems if stem.lstrip("-").isdigit()]
|
|
128
|
+
if len(set(numeric_ids)) != len(numeric_ids) or any(value < 0 for value in numeric_ids):
|
|
129
|
+
raise ValueError(f"Ambiguous or negative timestep identifiers in {directory}.")
|
|
130
|
+
return stems
|
|
131
|
+
|
|
132
|
+
@classmethod
|
|
133
|
+
def _load_blocks(cls, directory: Path, num_features: int) -> List[List[Any]]:
|
|
134
|
+
if not directory.exists():
|
|
135
|
+
raise FileNotFoundError(f"Timeline directory not found: {directory}")
|
|
136
|
+
|
|
137
|
+
files = sorted(directory.glob("*.bin"), key=lambda path: cls._sort_key(path.stem))
|
|
138
|
+
return [cls._read_bin_file(path, num_features) for path in files]
|
|
139
|
+
|
|
140
|
+
@staticmethod
|
|
141
|
+
def _sort_key(stem: str) -> Any:
|
|
142
|
+
try:
|
|
143
|
+
return (0, int(stem))
|
|
144
|
+
except ValueError:
|
|
145
|
+
return (1, stem)
|
|
146
|
+
|
|
147
|
+
@staticmethod
|
|
148
|
+
def _read_bin_file(path: Path, num_features: int) -> List[Any]:
|
|
149
|
+
payload = path.read_bytes()
|
|
150
|
+
values_per_record = num_features + 1
|
|
151
|
+
record_format = "<" + "f" * values_per_record
|
|
152
|
+
record_size = struct.calcsize(record_format)
|
|
153
|
+
if record_size <= 0 or len(payload) % record_size != 0:
|
|
154
|
+
raise ValueError(f"Unexpected record layout in {path}")
|
|
155
|
+
|
|
156
|
+
samples = []
|
|
157
|
+
for record in struct.iter_unpack(record_format, payload):
|
|
158
|
+
features = tuple(float(value) for value in record[:-1])
|
|
159
|
+
label_value = record[-1]
|
|
160
|
+
if not all(math.isfinite(value) for value in features):
|
|
161
|
+
raise ValueError(f"Non-finite feature value in {path}.")
|
|
162
|
+
if not math.isfinite(label_value) or label_value < 0 or not label_value.is_integer():
|
|
163
|
+
raise ValueError(f"Labels in {path} must be finite nonnegative integers.")
|
|
164
|
+
label = int(label_value)
|
|
165
|
+
samples.append((features, label))
|
|
166
|
+
return samples
|
retrainopt/cli.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""CLI entry point for RetrainOpt."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import re
|
|
9
|
+
from typing import List
|
|
10
|
+
|
|
11
|
+
from .adapters import BinaryTimelineAdapter, PrototypeModelAdapter, SklearnModelAdapter
|
|
12
|
+
from .config import PlannerConfig
|
|
13
|
+
from .planner import RetrainingPlanner
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _parse_n_choice(value: str) -> int | float:
|
|
17
|
+
"""Preserve integer-count versus decimal-fraction command-line intent."""
|
|
18
|
+
try:
|
|
19
|
+
if re.fullmatch(r"[+-]?\d+", value):
|
|
20
|
+
count = int(value)
|
|
21
|
+
if count >= 1:
|
|
22
|
+
return count
|
|
23
|
+
else:
|
|
24
|
+
fraction = float(value)
|
|
25
|
+
if math.isfinite(fraction) and 0 < fraction <= 1:
|
|
26
|
+
return fraction
|
|
27
|
+
except ValueError:
|
|
28
|
+
pass
|
|
29
|
+
raise argparse.ArgumentTypeError("Use a positive integer sample count (1) or a decimal fraction in (0, 1] (1.0 means all samples).")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
33
|
+
parser = argparse.ArgumentParser(
|
|
34
|
+
prog="retrainopt",
|
|
35
|
+
description="RetrainOpt: Cost-aware retraining planning",
|
|
36
|
+
)
|
|
37
|
+
subparsers = parser.add_subparsers(dest="command")
|
|
38
|
+
|
|
39
|
+
plan_parser = subparsers.add_parser(
|
|
40
|
+
"plan",
|
|
41
|
+
help="Plan a retraining schedule for a train/test binary timeline directory.",
|
|
42
|
+
)
|
|
43
|
+
plan_parser.add_argument("--dataset-dir", required=True, help="Path containing train/*.bin and test/*.bin.")
|
|
44
|
+
plan_parser.add_argument("--num-features", type=int, default=None, help="Feature count if metadata.json is absent.")
|
|
45
|
+
plan_parser.add_argument("--method", choices=["exact", "bayes"], default="exact")
|
|
46
|
+
plan_parser.add_argument(
|
|
47
|
+
"--bayes-backend",
|
|
48
|
+
choices=["lightweight", "botorch"],
|
|
49
|
+
default="lightweight",
|
|
50
|
+
help="Surrogate backend used when --method=bayes.",
|
|
51
|
+
)
|
|
52
|
+
plan_parser.add_argument(
|
|
53
|
+
"--model",
|
|
54
|
+
choices=["prototype", "logistic_regression", "random_forest", "gradient_boosting"],
|
|
55
|
+
default="prototype",
|
|
56
|
+
help="Built-in model factory for CLI usage.",
|
|
57
|
+
)
|
|
58
|
+
plan_parser.add_argument(
|
|
59
|
+
"--n-choice",
|
|
60
|
+
dest="n_choices",
|
|
61
|
+
action="append",
|
|
62
|
+
type=_parse_n_choice,
|
|
63
|
+
default=None,
|
|
64
|
+
help="Repeat for integer sample counts or decimal fractions in (0, 1]; 1 means one sample, 1.0 means all.",
|
|
65
|
+
)
|
|
66
|
+
plan_parser.add_argument("--min-train-size", type=int, default=1)
|
|
67
|
+
plan_parser.add_argument("--random-seed", type=int, default=42)
|
|
68
|
+
plan_parser.add_argument("--time-cost-weight", type=float, default=1.0)
|
|
69
|
+
plan_parser.add_argument("--loss-cost-weight", type=float, default=1.0)
|
|
70
|
+
plan_parser.add_argument("--json", action="store_true", help="Emit the result as JSON.")
|
|
71
|
+
return parser
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def main(argv: List[str] | None = None) -> int:
|
|
75
|
+
parser = build_parser()
|
|
76
|
+
args = parser.parse_args(argv)
|
|
77
|
+
if args.command != "plan":
|
|
78
|
+
parser.print_help()
|
|
79
|
+
return 0
|
|
80
|
+
|
|
81
|
+
timeline = BinaryTimelineAdapter(args.dataset_dir, num_features=args.num_features)
|
|
82
|
+
config = PlannerConfig(
|
|
83
|
+
n_choices=args.n_choices or (0.25, 0.5, 1.0),
|
|
84
|
+
min_train_size=args.min_train_size,
|
|
85
|
+
random_seed=args.random_seed,
|
|
86
|
+
time_cost_weight=args.time_cost_weight,
|
|
87
|
+
loss_cost_weight=args.loss_cost_weight,
|
|
88
|
+
bayes_backend=args.bayes_backend,
|
|
89
|
+
)
|
|
90
|
+
model_adapter = _build_model_adapter(args.model, args.random_seed)
|
|
91
|
+
planner = RetrainingPlanner(timeline, model_adapter, config=config)
|
|
92
|
+
result = planner.plan(method=args.method)
|
|
93
|
+
|
|
94
|
+
if args.json:
|
|
95
|
+
print(json.dumps(result.to_dict(), indent=2, allow_nan=False))
|
|
96
|
+
else:
|
|
97
|
+
print(f"RetrainOpt method={result.method} total_cost={result.total_cost:.6f}")
|
|
98
|
+
for step, decision in enumerate(result.decisions):
|
|
99
|
+
if decision.action.value == "keep":
|
|
100
|
+
print(f"t={step}: keep")
|
|
101
|
+
else:
|
|
102
|
+
print(f"t={step}: retrain n={decision.n_samples}")
|
|
103
|
+
return 0
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _build_sklearn_factory(model_name: str, random_seed: int):
|
|
107
|
+
def _factory():
|
|
108
|
+
try:
|
|
109
|
+
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
|
|
110
|
+
from sklearn.linear_model import LogisticRegression
|
|
111
|
+
except ImportError as exc: # pragma: no cover - optional dependency
|
|
112
|
+
raise RuntimeError(
|
|
113
|
+
"The RetrainOpt CLI model presets require the `sklearn` extra."
|
|
114
|
+
) from exc
|
|
115
|
+
|
|
116
|
+
if model_name == "logistic_regression":
|
|
117
|
+
return LogisticRegression(max_iter=1000, random_state=random_seed)
|
|
118
|
+
if model_name == "random_forest":
|
|
119
|
+
return RandomForestClassifier(n_estimators=100, random_state=random_seed)
|
|
120
|
+
if model_name == "gradient_boosting":
|
|
121
|
+
return GradientBoostingClassifier(random_state=random_seed)
|
|
122
|
+
raise ValueError(f"Unsupported CLI model preset: {model_name}")
|
|
123
|
+
|
|
124
|
+
return _factory
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _build_model_adapter(model_name: str, random_seed: int):
|
|
128
|
+
if model_name == "prototype":
|
|
129
|
+
return PrototypeModelAdapter()
|
|
130
|
+
return SklearnModelAdapter(_build_sklearn_factory(model_name, random_seed))
|
retrainopt/config.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Configuration objects for RetrainOpt."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
import copy
|
|
7
|
+
import math
|
|
8
|
+
from numbers import Integral, Real
|
|
9
|
+
from typing import Any, Dict, List, Optional, Sequence, Union
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
NumberLike = Union[int, float]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class PlannerConfig:
|
|
17
|
+
"""User-facing planner configuration.
|
|
18
|
+
|
|
19
|
+
``n_choices`` accepts float fractions in ``(0, 1]`` or integer sample
|
|
20
|
+
counts ``>= 1``. Thus ``1`` means one sample and ``1.0`` means all samples.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
n_choices: Sequence[NumberLike] = field(default_factory=lambda: (0.25, 0.5, 1.0))
|
|
24
|
+
min_train_size: int = 1
|
|
25
|
+
time_cost_weight: float = 1.0
|
|
26
|
+
loss_cost_weight: float = 1.0
|
|
27
|
+
random_seed: int = 42
|
|
28
|
+
|
|
29
|
+
# If ``None``, the planner synthesizes the initial model by training on the
|
|
30
|
+
# first timestep with ``initial_model_samples`` or the full first block.
|
|
31
|
+
initial_loss: Optional[float] = None
|
|
32
|
+
initial_model_samples: Optional[int] = None
|
|
33
|
+
|
|
34
|
+
# Approximate "bayes" mode controls.
|
|
35
|
+
bayes_iterations: int = 5
|
|
36
|
+
bayes_initial_samples: int = 6
|
|
37
|
+
bayes_candidates_per_iteration: int = 1
|
|
38
|
+
bayes_selection_strategy: str = "sequence"
|
|
39
|
+
bayes_anchor_steps: Sequence[float] = field(default_factory=lambda: (0.0, 0.5, 1.0))
|
|
40
|
+
acquisition: str = "expected"
|
|
41
|
+
bayes_backend: str = "lightweight"
|
|
42
|
+
exploration_beta: float = 1.0
|
|
43
|
+
improvement_xi: float = 0.01
|
|
44
|
+
surrogate_k: int = 5
|
|
45
|
+
surrogate_uncertainty_floor: float = 0.05
|
|
46
|
+
bayes_backend_options: Dict[str, Any] = field(default_factory=dict)
|
|
47
|
+
record_bayes_true_history: bool = False
|
|
48
|
+
|
|
49
|
+
def __post_init__(self) -> None:
|
|
50
|
+
choices = tuple(self.n_choices)
|
|
51
|
+
if not choices:
|
|
52
|
+
raise ValueError("n_choices must contain at least one sample count or fraction.")
|
|
53
|
+
for choice in choices:
|
|
54
|
+
if isinstance(choice, bool) or not isinstance(choice, Real):
|
|
55
|
+
raise ValueError("n_choices must contain positive integers or float fractions in (0, 1].")
|
|
56
|
+
if isinstance(choice, Integral):
|
|
57
|
+
if choice < 1:
|
|
58
|
+
raise ValueError("Integer n_choices must be positive sample counts.")
|
|
59
|
+
elif not math.isfinite(choice) or not 0 < choice <= 1:
|
|
60
|
+
raise ValueError("Float n_choices must be finite fractions in (0, 1]; use an integer for a sample count.")
|
|
61
|
+
object.__setattr__(self, "n_choices", choices)
|
|
62
|
+
for name, minimum in (
|
|
63
|
+
("min_train_size", 1), ("random_seed", 0), ("bayes_iterations", 0),
|
|
64
|
+
("bayes_initial_samples", 1), ("bayes_candidates_per_iteration", 1), ("surrogate_k", 1),
|
|
65
|
+
):
|
|
66
|
+
value = getattr(self, name)
|
|
67
|
+
if isinstance(value, bool) or not isinstance(value, Integral) or value < minimum:
|
|
68
|
+
raise ValueError(f"{name} must be an integer >= {minimum}.")
|
|
69
|
+
if self.initial_model_samples is not None and (
|
|
70
|
+
isinstance(self.initial_model_samples, bool)
|
|
71
|
+
or not isinstance(self.initial_model_samples, Integral)
|
|
72
|
+
or self.initial_model_samples < 1
|
|
73
|
+
):
|
|
74
|
+
raise ValueError("initial_model_samples must be a positive integer.")
|
|
75
|
+
for name in ("time_cost_weight", "loss_cost_weight", "exploration_beta", "improvement_xi", "surrogate_uncertainty_floor", "initial_loss"):
|
|
76
|
+
value = getattr(self, name)
|
|
77
|
+
if value is None and name == "initial_loss":
|
|
78
|
+
continue
|
|
79
|
+
if isinstance(value, bool) or not isinstance(value, Real) or not math.isfinite(value) or value < 0:
|
|
80
|
+
raise ValueError(f"{name} must be finite and nonnegative.")
|
|
81
|
+
anchors = tuple(self.bayes_anchor_steps)
|
|
82
|
+
if any(isinstance(value, bool) or not isinstance(value, Real) or not math.isfinite(value) or not 0 <= value <= 1 for value in anchors):
|
|
83
|
+
raise ValueError("bayes_anchor_steps must contain finite fractions in [0, 1].")
|
|
84
|
+
object.__setattr__(self, "bayes_anchor_steps", anchors)
|
|
85
|
+
if not isinstance(self.bayes_backend_options, dict):
|
|
86
|
+
raise ValueError("bayes_backend_options must be a dictionary.")
|
|
87
|
+
object.__setattr__(self, "bayes_backend_options", copy.deepcopy(self.bayes_backend_options))
|
|
88
|
+
|
|
89
|
+
def resolve_n_choices(self, dataset_size: int) -> List[int]:
|
|
90
|
+
"""Resolve configured sample choices for a concrete train block size."""
|
|
91
|
+
if isinstance(dataset_size, bool) or not isinstance(dataset_size, Integral) or dataset_size < 0:
|
|
92
|
+
raise ValueError("dataset_size must be a nonnegative integer.")
|
|
93
|
+
if dataset_size < self.min_train_size:
|
|
94
|
+
return []
|
|
95
|
+
|
|
96
|
+
choices = set()
|
|
97
|
+
for choice in self.n_choices:
|
|
98
|
+
if isinstance(choice, Integral):
|
|
99
|
+
n_value = min(dataset_size, choice)
|
|
100
|
+
else:
|
|
101
|
+
n_value = int(dataset_size * float(choice))
|
|
102
|
+
|
|
103
|
+
if n_value >= self.min_train_size:
|
|
104
|
+
choices.add(max(1, n_value))
|
|
105
|
+
|
|
106
|
+
return sorted(choices)
|