authtracker 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- authtracker/__init__.py +16 -0
- authtracker/backends/__init__.py +95 -0
- authtracker/backends/base.py +136 -0
- authtracker/backends/local.py +200 -0
- authtracker/backends/mlflow_backend.py +310 -0
- authtracker/backends/resilient.py +329 -0
- authtracker/backends/wandb_backend.py +235 -0
- authtracker/bundle.py +887 -0
- authtracker/capture.py +175 -0
- authtracker/cli.py +51 -0
- authtracker/configfile.py +84 -0
- authtracker/dataset.py +98 -0
- authtracker/errors.py +18 -0
- authtracker/registry.py +118 -0
- authtracker/run.py +346 -0
- authtracker/settings.py +171 -0
- authtracker/simple.py +241 -0
- authtracker/testing.py +82 -0
- authtracker/utils.py +321 -0
- authtracker/watch.py +59 -0
- authtracker-0.0.1.dist-info/METADATA +215 -0
- authtracker-0.0.1.dist-info/RECORD +25 -0
- authtracker-0.0.1.dist-info/WHEEL +4 -0
- authtracker-0.0.1.dist-info/entry_points.txt +2 -0
- authtracker-0.0.1.dist-info/licenses/LICENSE +202 -0
authtracker/run.py
ADDED
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
"""Tracking: model -> experiment -> run hierarchy.
|
|
2
|
+
|
|
3
|
+
with authtracker.run(model="model1", experiment="churn-v2") as r: # parent run
|
|
4
|
+
with r.child("lr0.01") as c: # child run
|
|
5
|
+
c.log_params({...}); c.log_eval(f1=0.9)
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import dataclasses
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
import shutil
|
|
13
|
+
import tempfile
|
|
14
|
+
import traceback
|
|
15
|
+
from contextlib import contextmanager
|
|
16
|
+
from typing import Any, Dict, Iterable, Iterator, Mapping, Optional, Sequence, Union
|
|
17
|
+
|
|
18
|
+
from .backends import Backend, get_backend
|
|
19
|
+
from . import bundle, capture, utils
|
|
20
|
+
from .backends.base import TAG_LEVEL
|
|
21
|
+
from .errors import BackendError, UnsupportedError, ValidationError
|
|
22
|
+
from .settings import Settings, get_settings
|
|
23
|
+
from .utils import capture_env_tags, requirements_text
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _changed_params(existing: Mapping[str, Any], params: Mapping[str, Any]) -> Dict[str, tuple]:
|
|
27
|
+
return {k: (existing[k], str(v)) for k, v in params.items() if k in existing and existing[k] != str(v)}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _params_conflict(changed: Mapping[str, tuple]) -> ValidationError:
|
|
31
|
+
details = ", ".join(f"{k} was {old} and is now {new}" for k, (old, new) in changed.items())
|
|
32
|
+
return ValidationError(
|
|
33
|
+
f"A resumed run cannot change parameters it already has: {details}. Keep the old value, or "
|
|
34
|
+
"start a new run instead of resuming (resume=False).")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Run:
|
|
38
|
+
def __init__(self, backend: Backend, run_id: str, tags: Dict[str, str], settings: Settings,
|
|
39
|
+
resume_n: int = 0):
|
|
40
|
+
self._backend = backend
|
|
41
|
+
self.id = run_id
|
|
42
|
+
self.tags = tags
|
|
43
|
+
self._settings = settings
|
|
44
|
+
self.resume_n = resume_n # 0 for a new run; 1, 2, ... for the first, second ... resume
|
|
45
|
+
|
|
46
|
+
def _named(self, name: str) -> str:
|
|
47
|
+
"""A file name for this part of the run. Resumed runs add `.resume1`, `.resume2` ... so the files of
|
|
48
|
+
earlier parts (the log, the code) are kept instead of overwritten: `output.log` -> `output.resume1.log`."""
|
|
49
|
+
if not self.resume_n:
|
|
50
|
+
return name
|
|
51
|
+
base, extension = os.path.splitext(name)
|
|
52
|
+
return f"{base}.resume{self.resume_n}{extension}"
|
|
53
|
+
|
|
54
|
+
# logging
|
|
55
|
+
def set_tags(self, tags: Mapping[str, Any]) -> None:
|
|
56
|
+
clean = {k: str(v) for k, v in tags.items()}
|
|
57
|
+
self._backend.set_tags(self.id, clean)
|
|
58
|
+
self.tags.update(clean) # runs started from this one afterwards inherit them
|
|
59
|
+
|
|
60
|
+
def log_params(self, params: Mapping[str, Any]) -> None:
|
|
61
|
+
params = dict(params)
|
|
62
|
+
if self.resume_n: # a resumed run keeps the parameters it already has
|
|
63
|
+
existing = self._backend.get_run(self.id)["params"]
|
|
64
|
+
changed = _changed_params(existing, params)
|
|
65
|
+
if changed:
|
|
66
|
+
raise _params_conflict(changed)
|
|
67
|
+
params = {k: v for k, v in params.items() if k not in existing}
|
|
68
|
+
if params:
|
|
69
|
+
self._backend.log_params(self.id, params)
|
|
70
|
+
|
|
71
|
+
def log_param(self, key: str, value: Any) -> None:
|
|
72
|
+
self.log_params({key: value})
|
|
73
|
+
|
|
74
|
+
def log_metrics(self, metrics: Mapping[str, float], step: Optional[int] = None) -> None:
|
|
75
|
+
self._backend.log_metrics(self.id, dict(metrics), step)
|
|
76
|
+
|
|
77
|
+
def log_metric(self, key: str, value: float, step: Optional[int] = None) -> None:
|
|
78
|
+
self.log_metrics({key: value}, step)
|
|
79
|
+
|
|
80
|
+
def check_metric_names(self, names: Iterable[str]) -> None:
|
|
81
|
+
"""Raise if AUTHTRACKER_METRIC_NAMES is set and `names` contains anything else."""
|
|
82
|
+
allowed = self._settings.metric_names
|
|
83
|
+
if allowed:
|
|
84
|
+
bad = sorted(set(names) - set(allowed))
|
|
85
|
+
if bad:
|
|
86
|
+
raise ValidationError(f"Non-standard metric names {bad}; allowed: {list(allowed)}")
|
|
87
|
+
|
|
88
|
+
def log_eval(self, **metrics: float) -> None:
|
|
89
|
+
"""Log final metrics, enforcing AUTHTRACKER_METRIC_NAMES when it is set."""
|
|
90
|
+
self.check_metric_names(metrics)
|
|
91
|
+
self.log_metrics(metrics)
|
|
92
|
+
|
|
93
|
+
def log_artifact(self, path: str) -> None:
|
|
94
|
+
self._backend.log_artifact(self.id, path)
|
|
95
|
+
|
|
96
|
+
def log_text(self, text: str, name: str) -> None:
|
|
97
|
+
"""Save `text` as an artifact called `name` (e.g. a training log: 'train.log')."""
|
|
98
|
+
with tempfile.TemporaryDirectory() as d:
|
|
99
|
+
path = os.path.join(d, os.path.basename(name))
|
|
100
|
+
with open(path, "w") as f:
|
|
101
|
+
f.write(text)
|
|
102
|
+
self.log_artifact(path)
|
|
103
|
+
|
|
104
|
+
def log_artifacts(self, source: Union[str, Sequence[str]]) -> None:
|
|
105
|
+
"""Log a bundle: every file directly inside a directory, or a list of file paths.
|
|
106
|
+
|
|
107
|
+
Files keep their names (no sub-folders), so names inside a bundle must be unique.
|
|
108
|
+
"""
|
|
109
|
+
if isinstance(source, str):
|
|
110
|
+
if not os.path.isdir(source):
|
|
111
|
+
raise ValidationError(f"{source!r} is not a directory")
|
|
112
|
+
paths = sorted(os.path.join(source, n) for n in os.listdir(source)
|
|
113
|
+
if os.path.isfile(os.path.join(source, n)))
|
|
114
|
+
else:
|
|
115
|
+
paths = list(source)
|
|
116
|
+
names = [os.path.basename(p) for p in paths]
|
|
117
|
+
dupes = sorted({n for n in names if names.count(n) > 1})
|
|
118
|
+
if dupes:
|
|
119
|
+
raise ValidationError(f"Duplicate file names in bundle: {dupes}")
|
|
120
|
+
for p in paths:
|
|
121
|
+
self.log_artifact(p)
|
|
122
|
+
|
|
123
|
+
def restore(self, name: str, folder: Optional[str] = None) -> Optional[str]:
|
|
124
|
+
"""Download the file `name` saved on this run (for example a checkpoint) and return its path, or None
|
|
125
|
+
if there is no such file. `folder` is where to put it (default: a new temporary folder)."""
|
|
126
|
+
folder = folder or tempfile.mkdtemp(prefix="authtracker-restore-")
|
|
127
|
+
os.makedirs(folder, exist_ok=True)
|
|
128
|
+
return self._backend.download_artifact(self.id, name, folder)
|
|
129
|
+
|
|
130
|
+
def log_model(self, model: Any, flavor: Optional[str] = None, **kwargs: Any) -> str:
|
|
131
|
+
"""Log a model; call it inside the `with` block of the run that owns it.
|
|
132
|
+
|
|
133
|
+
flavor="bundle" stores a PyTorch model as a plain folder (architecture file, weights, sample
|
|
134
|
+
inference script, requirements): see authtracker.bundle.
|
|
135
|
+
"""
|
|
136
|
+
if flavor in (bundle.FLAVOR, bundle.PICKLE_FLAVOR):
|
|
137
|
+
builder = bundle.build if flavor == bundle.FLAVOR else bundle.build_pickle
|
|
138
|
+
container = {k[len("container_"):]: v for k, v in self.tags.items() if k.startswith("container_")}
|
|
139
|
+
with tempfile.TemporaryDirectory(prefix="authtracker-bundle-") as folder:
|
|
140
|
+
builder(model, folder, context={"container": container or None}, **kwargs)
|
|
141
|
+
return self._backend.log_model_dir(self.id, folder)
|
|
142
|
+
return self._backend.log_model(self.id, model, flavor, **kwargs)
|
|
143
|
+
|
|
144
|
+
def child(self, name: str, params: Optional[Mapping[str, Any]] = None, resume: bool = False):
|
|
145
|
+
"""Start a nested child run (the 'run' level). With resume=True, continue the latest earlier run
|
|
146
|
+
of this name under this experiment run, or start a new one if there is none."""
|
|
147
|
+
inherited = {k: v for k, v in self.tags.items() if k != TAG_LEVEL and not k.startswith("resume")}
|
|
148
|
+
return _managed(self._backend, self._settings, self.tags["experiment"], name, inherited,
|
|
149
|
+
parent=self, level="run", params=params, resume=resume)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _periodic_upload(backend: Backend, run_id: str):
|
|
153
|
+
"""Best-effort upload of the log while the run is going. It bypasses the offline queue on
|
|
154
|
+
purpose: a queued copy of a growing log every minute would pile up, and the final upload at the
|
|
155
|
+
end of the run is the one that always happens."""
|
|
156
|
+
inner = getattr(backend, "inner", backend)
|
|
157
|
+
resolve = getattr(backend, "_res", lambda rid: rid)
|
|
158
|
+
|
|
159
|
+
def upload(path: str) -> None:
|
|
160
|
+
rid = resolve(run_id)
|
|
161
|
+
if rid is None or str(rid).startswith("offline-"):
|
|
162
|
+
return # the run is not on the server yet
|
|
163
|
+
inner.log_artifact(rid, path)
|
|
164
|
+
|
|
165
|
+
return upload
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _save_captured_log(run: Run, log_capture, log_dir) -> None:
|
|
169
|
+
"""Stop copying console output and save it to the run as output.log (best effort)."""
|
|
170
|
+
if log_capture is None:
|
|
171
|
+
return
|
|
172
|
+
try:
|
|
173
|
+
capture.stop(log_capture)
|
|
174
|
+
if os.path.getsize(log_capture.path) > 0:
|
|
175
|
+
run.log_artifact(log_capture.path)
|
|
176
|
+
except Exception as e:
|
|
177
|
+
logging.getLogger("authtracker").warning("Could not save output.log: %s", e)
|
|
178
|
+
finally:
|
|
179
|
+
shutil.rmtree(log_dir, ignore_errors=True)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _validate(settings: Settings, tags: Dict[str, str], model: str) -> None:
|
|
183
|
+
if settings.allowed_models and model not in settings.allowed_models:
|
|
184
|
+
raise ValidationError(
|
|
185
|
+
f"Unknown model {model!r}; allowed: {list(settings.allowed_models)}")
|
|
186
|
+
missing = [t for t in settings.required_tags if not tags.get(t)]
|
|
187
|
+
if missing:
|
|
188
|
+
raise ValidationError(f"Missing required tags: {missing}")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _save_code(run: Run, snapshot: Mapping[str, Any]) -> None:
|
|
192
|
+
"""Store the uncommitted changes and the running script with the experiment run."""
|
|
193
|
+
patch, script = snapshot.get("changes", {}), snapshot.get("script", {})
|
|
194
|
+
try:
|
|
195
|
+
if patch.get("patch"):
|
|
196
|
+
run.log_text(patch["patch"], run._named(utils.PATCH_NAME))
|
|
197
|
+
if script.get("text"):
|
|
198
|
+
run.log_text(script["text"], run._named(f"script_{script['name']}"))
|
|
199
|
+
except Exception as e:
|
|
200
|
+
logging.getLogger("authtracker").warning("Could not save the code: %s", e)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _code_snapshot() -> Dict[str, Any]:
|
|
204
|
+
"""What to store about the code, and tags that say what was (and was not) stored and why."""
|
|
205
|
+
changes, script = utils.code_changes(), utils.script_snapshot()
|
|
206
|
+
tags: Dict[str, str] = {"code_changes": changes["status"], "code_script": script["status"]}
|
|
207
|
+
if script.get("repo_path"):
|
|
208
|
+
tags["git_script"] = script["repo_path"].replace(os.sep, "/")
|
|
209
|
+
if changes["untracked_skipped"]:
|
|
210
|
+
tags["code_changes_untracked_skipped"] = str(changes["untracked_skipped"])
|
|
211
|
+
for label, part in (("changes", changes), ("script", script)):
|
|
212
|
+
if part["status"].startswith("skipped"):
|
|
213
|
+
logging.getLogger("authtracker").warning(
|
|
214
|
+
"The code %s was not stored (%s). Pass save_code=False to at.init(...) to stop this check.",
|
|
215
|
+
label, part["status"].replace("skipped: ", ""))
|
|
216
|
+
return {"changes": changes, "script": script, "tags": tags}
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _find_earlier(backend: Backend, run_name: str, tags: Dict[str, str], parent: Optional[Run]) -> Optional[str]:
|
|
220
|
+
"""The newest earlier run to continue, or None. Only an unsupported backend is an error: if the server
|
|
221
|
+
cannot be reached to look, a new run is started (and queued) instead of stopping the training."""
|
|
222
|
+
wanted = {k: tags[k] for k in ("model", "experiment", "owner", TAG_LEVEL) if k in tags}
|
|
223
|
+
try:
|
|
224
|
+
return backend.find_run(tags["model"], run_name, wanted, parent.id if parent else None)
|
|
225
|
+
except UnsupportedError:
|
|
226
|
+
raise
|
|
227
|
+
except BackendError as e:
|
|
228
|
+
logging.getLogger("authtracker").warning(
|
|
229
|
+
"Could not look for an earlier run to resume (%s). Starting a new run.", e)
|
|
230
|
+
return None
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
_ENVIRONMENT_TAGS = ("python_version", "platform", "hostname")
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _tag_resume(run: Run, old: Mapping[str, str], new: Mapping[str, str], n: int) -> None:
|
|
237
|
+
"""Mark a run as resumed, and keep what is different about this part (its commit, its image) next to the
|
|
238
|
+
original instead of replacing it: `git_sha` stays, `resume1_git_sha` is added."""
|
|
239
|
+
updates = {"resumed": "true", "resume_count": str(n)}
|
|
240
|
+
for key, value in new.items():
|
|
241
|
+
if key in ("resumed", "resume_count") or key.startswith("resume"):
|
|
242
|
+
continue
|
|
243
|
+
if key not in old:
|
|
244
|
+
updates[key] = value
|
|
245
|
+
elif old[key] != value and (key.startswith(("git_", "code_", "container_")) or key in _ENVIRONMENT_TAGS):
|
|
246
|
+
updates[f"resume{n}_{key}"] = value
|
|
247
|
+
run.set_tags(updates)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
@contextmanager
|
|
251
|
+
def _managed(backend: Backend, settings: Settings, experiment_name: str, run_name: str,
|
|
252
|
+
tags: Dict[str, str], parent: Optional[Run], level: str,
|
|
253
|
+
params: Optional[Mapping[str, Any]] = None, code: Optional[Mapping[str, Any]] = None,
|
|
254
|
+
resume: bool = False) -> Iterator[Run]:
|
|
255
|
+
full_tags = {**tags, TAG_LEVEL: level}
|
|
256
|
+
earlier = _find_earlier(backend, run_name, full_tags, parent) if resume else None
|
|
257
|
+
if earlier:
|
|
258
|
+
old = dict(backend.get_run(earlier)["tags"])
|
|
259
|
+
changed = _changed_params(backend.get_run(earlier)["params"], params or {})
|
|
260
|
+
if changed: # refuse BEFORE touching the old run: it stays as it was
|
|
261
|
+
raise _params_conflict(changed)
|
|
262
|
+
backend.resume_run(earlier, parent.id if parent else None)
|
|
263
|
+
run_id = earlier
|
|
264
|
+
n = int(old.get("resume_count") or 0) + 1
|
|
265
|
+
run = Run(backend, run_id, {**old, **full_tags}, settings, resume_n=n)
|
|
266
|
+
_tag_resume(run, old, full_tags, n)
|
|
267
|
+
else:
|
|
268
|
+
run_id = backend.start_run(tags["model"], run_name, full_tags,
|
|
269
|
+
parent.id if parent else None)
|
|
270
|
+
run = Run(backend, run_id, full_tags, settings)
|
|
271
|
+
status = "FINISHED"
|
|
272
|
+
log_capture, log_dir = None, None
|
|
273
|
+
if settings.capture_logs and not getattr(backend, "captures_console", False):
|
|
274
|
+
try:
|
|
275
|
+
log_dir = tempfile.mkdtemp(prefix="authtracker-")
|
|
276
|
+
log_capture = capture.start(os.path.join(log_dir, run._named("output.log")),
|
|
277
|
+
periodic=_periodic_upload(backend, run_id),
|
|
278
|
+
interval=settings.log_upload_interval)
|
|
279
|
+
except Exception:
|
|
280
|
+
log_capture = None # capturing is a convenience: never stop the run for it
|
|
281
|
+
try:
|
|
282
|
+
if level == "experiment" and settings.capture_env:
|
|
283
|
+
with tempfile.TemporaryDirectory() as d:
|
|
284
|
+
path = os.path.join(d, run._named("requirements.txt"))
|
|
285
|
+
with open(path, "w") as f:
|
|
286
|
+
f.write(requirements_text())
|
|
287
|
+
run.log_artifact(path)
|
|
288
|
+
if level == "experiment" and code:
|
|
289
|
+
_save_code(run, code)
|
|
290
|
+
if params:
|
|
291
|
+
run.log_params(params)
|
|
292
|
+
yield run
|
|
293
|
+
except BaseException as e:
|
|
294
|
+
status = "KILLED" if isinstance(e, KeyboardInterrupt) else "FAILED"
|
|
295
|
+
try:
|
|
296
|
+
with tempfile.TemporaryDirectory() as d:
|
|
297
|
+
path = os.path.join(d, run._named("error.txt"))
|
|
298
|
+
with open(path, "w") as f:
|
|
299
|
+
f.write("".join(traceback.format_exception(type(e), e, e.__traceback__)))
|
|
300
|
+
run.log_artifact(path)
|
|
301
|
+
except Exception:
|
|
302
|
+
pass # never mask the original error
|
|
303
|
+
raise
|
|
304
|
+
finally:
|
|
305
|
+
_save_captured_log(run, log_capture, log_dir)
|
|
306
|
+
backend.end_run(run_id, status)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def run(model: str, experiment: str, *, owner: Optional[str] = None,
|
|
310
|
+
dataset_version: Optional[str] = None, tags: Optional[Mapping[str, str]] = None,
|
|
311
|
+
save_code: Optional[bool] = None, image: Optional[str] = None, system_metrics: Optional[bool] = None,
|
|
312
|
+
resume: bool = False, params: Optional[Mapping[str, Any]] = None,
|
|
313
|
+
settings: Optional[Settings] = None, backend: Optional[Backend] = None):
|
|
314
|
+
"""Start the parent run for `experiment` under `model`. Use as a context manager.
|
|
315
|
+
|
|
316
|
+
resume: continue the latest earlier experiment run of this model, experiment and owner, or start a
|
|
317
|
+
new one if there is none. A resumed run keeps its parameters and continues its metric steps.
|
|
318
|
+
|
|
319
|
+
save_code: store the uncommitted changes and the script with the run (default: on).
|
|
320
|
+
image: the container image this runs in, ideally with its digest ("registry/app@sha256:...").
|
|
321
|
+
system_metrics: record CPU, memory and GPU use while the run is open (default: on).
|
|
322
|
+
These override the defaults from the settings, for this run only.
|
|
323
|
+
"""
|
|
324
|
+
s = settings or get_settings()
|
|
325
|
+
overrides = {k: v for k, v in (("save_code", save_code), ("image", image),
|
|
326
|
+
("system_metrics", system_metrics)) if v is not None}
|
|
327
|
+
if overrides:
|
|
328
|
+
s = dataclasses.replace(s, **overrides)
|
|
329
|
+
all_tags: Dict[str, str] = {**(tags or {}), "model": model, "experiment": experiment,
|
|
330
|
+
"owner": owner or s.owner}
|
|
331
|
+
if dataset_version:
|
|
332
|
+
all_tags["dataset_version"] = dataset_version
|
|
333
|
+
_validate(s, all_tags, model)
|
|
334
|
+
if s.capture_env:
|
|
335
|
+
for k, v in capture_env_tags().items():
|
|
336
|
+
all_tags.setdefault(k, v)
|
|
337
|
+
container = utils.container_info(s.image)
|
|
338
|
+
if container:
|
|
339
|
+
for key, value in container.items():
|
|
340
|
+
all_tags.setdefault(f"container_{key}", value)
|
|
341
|
+
code = _code_snapshot() if s.save_code else None
|
|
342
|
+
if code:
|
|
343
|
+
for k, v in code["tags"].items():
|
|
344
|
+
all_tags.setdefault(k, v)
|
|
345
|
+
return _managed(backend or get_backend(s), s, experiment, experiment, all_tags,
|
|
346
|
+
parent=None, level="experiment", params=params, code=code, resume=resume)
|
authtracker/settings.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Configuration, read entirely from AUTHTRACKER_* environment variables.
|
|
2
|
+
|
|
3
|
+
No config file is required. Everything has a default, and code can override
|
|
4
|
+
settings explicitly with ``authtracker.configure(Settings(...))``.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import getpass
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import warnings
|
|
12
|
+
from dataclasses import asdict, dataclass
|
|
13
|
+
from typing import Mapping, Optional, Tuple
|
|
14
|
+
|
|
15
|
+
PREFIX = "AUTHTRACKER_"
|
|
16
|
+
|
|
17
|
+
_TRUE = {"1", "true", "yes", "on"}
|
|
18
|
+
_FALSE = {"0", "false", "no", "off"}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ConfigError(ValueError):
|
|
22
|
+
"""An AUTHTRACKER_* variable has an invalid value."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _csv(value: str) -> Tuple[str, ...]:
|
|
26
|
+
return tuple(x.strip() for x in value.split(",") if x.strip())
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _bool(name: str, value: str) -> bool:
|
|
30
|
+
v = value.strip().lower()
|
|
31
|
+
if v in _TRUE:
|
|
32
|
+
return True
|
|
33
|
+
if v in _FALSE:
|
|
34
|
+
return False
|
|
35
|
+
raise ConfigError(f"{PREFIX}{name}={value!r} is not a boolean (use true/false)")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _float(name: str, value: str) -> float:
|
|
39
|
+
try:
|
|
40
|
+
return float(value)
|
|
41
|
+
except ValueError:
|
|
42
|
+
raise ConfigError(f"{PREFIX}{name}={value!r} is not a number") from None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _int(name: str, value: str) -> int:
|
|
46
|
+
try:
|
|
47
|
+
n = int(value)
|
|
48
|
+
except ValueError:
|
|
49
|
+
raise ConfigError(f"{PREFIX}{name}={value!r} is not an integer") from None
|
|
50
|
+
if n < 0:
|
|
51
|
+
raise ConfigError(f"{PREFIX}{name}={value!r} must be >= 0")
|
|
52
|
+
return n
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _default_owner() -> str:
|
|
56
|
+
try:
|
|
57
|
+
return getpass.getuser()
|
|
58
|
+
except Exception:
|
|
59
|
+
return "unknown"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class Settings:
|
|
64
|
+
backend: str = "mlflow"
|
|
65
|
+
tracking_uri: Optional[str] = None
|
|
66
|
+
registry_uri: Optional[str] = None
|
|
67
|
+
owner: str = ""
|
|
68
|
+
required_tags: Tuple[str, ...] = ()
|
|
69
|
+
allowed_models: Tuple[str, ...] = ()
|
|
70
|
+
metric_names: Tuple[str, ...] = ()
|
|
71
|
+
capture_env: bool = True
|
|
72
|
+
capture_logs: bool = True
|
|
73
|
+
save_code: bool = True
|
|
74
|
+
image: Optional[str] = None
|
|
75
|
+
log_upload_interval: float = 60.0
|
|
76
|
+
system_metrics: bool = True
|
|
77
|
+
promote_metric: Optional[str] = None
|
|
78
|
+
promote_min: Optional[float] = None
|
|
79
|
+
promote_must_beat_prod: bool = False
|
|
80
|
+
gated_aliases: Tuple[str, ...] = ("production",)
|
|
81
|
+
offline_queue: bool = True
|
|
82
|
+
queue_dir: Optional[str] = None
|
|
83
|
+
retries: int = 3
|
|
84
|
+
retry_wait: float = 1.0
|
|
85
|
+
|
|
86
|
+
@classmethod
|
|
87
|
+
def from_env(cls, environ: Optional[Mapping[str, str]] = None) -> "Settings":
|
|
88
|
+
env = os.environ if environ is None else environ
|
|
89
|
+
|
|
90
|
+
def get(name: str) -> Optional[str]:
|
|
91
|
+
v = env.get(PREFIX + name)
|
|
92
|
+
return v if v not in (None, "") else None
|
|
93
|
+
|
|
94
|
+
known = {
|
|
95
|
+
"BACKEND", "TRACKING_URI", "REGISTRY_URI", "OWNER", "REQUIRED_TAGS",
|
|
96
|
+
"ALLOWED_MODELS", "METRIC_NAMES", "CAPTURE_ENV", "CAPTURE_LOGS", "SAVE_CODE", "IMAGE", "LOG_UPLOAD_INTERVAL", "SYSTEM_METRICS", "PROMOTE_METRIC",
|
|
97
|
+
"PROMOTE_MIN", "PROMOTE_MUST_BEAT_PROD", "GATED_ALIASES",
|
|
98
|
+
"OFFLINE_QUEUE", "QUEUE_DIR", "RETRIES", "RETRY_WAIT",
|
|
99
|
+
}
|
|
100
|
+
for key in env:
|
|
101
|
+
if key.startswith(PREFIX) and key[len(PREFIX):] not in known:
|
|
102
|
+
warnings.warn(f"Unknown variable {key} (typo?)", stacklevel=2)
|
|
103
|
+
|
|
104
|
+
kw = {}
|
|
105
|
+
if (v := get("BACKEND")) is not None:
|
|
106
|
+
kw["backend"] = v.strip().lower()
|
|
107
|
+
# Fall back to MLflow's own variable so existing setups keep working.
|
|
108
|
+
uri = get("TRACKING_URI") or env.get("MLFLOW_TRACKING_URI") or None
|
|
109
|
+
if uri:
|
|
110
|
+
kw["tracking_uri"] = uri
|
|
111
|
+
if (v := get("REGISTRY_URI")) is not None:
|
|
112
|
+
kw["registry_uri"] = v
|
|
113
|
+
kw["owner"] = get("OWNER") or _default_owner()
|
|
114
|
+
if (v := get("REQUIRED_TAGS")) is not None:
|
|
115
|
+
kw["required_tags"] = _csv(v)
|
|
116
|
+
if (v := get("ALLOWED_MODELS")) is not None:
|
|
117
|
+
kw["allowed_models"] = _csv(v)
|
|
118
|
+
if (v := get("METRIC_NAMES")) is not None:
|
|
119
|
+
kw["metric_names"] = _csv(v)
|
|
120
|
+
if (v := get("CAPTURE_ENV")) is not None:
|
|
121
|
+
kw["capture_env"] = _bool("CAPTURE_ENV", v)
|
|
122
|
+
if (v := get("IMAGE")) is not None:
|
|
123
|
+
kw["image"] = v
|
|
124
|
+
if (v := get("SAVE_CODE")) is not None:
|
|
125
|
+
kw["save_code"] = _bool("SAVE_CODE", v)
|
|
126
|
+
if (v := get("LOG_UPLOAD_INTERVAL")) is not None:
|
|
127
|
+
kw["log_upload_interval"] = _float("LOG_UPLOAD_INTERVAL", v)
|
|
128
|
+
if kw["log_upload_interval"] < 0:
|
|
129
|
+
raise ConfigError(f"{PREFIX}LOG_UPLOAD_INTERVAL={v!r} must be 0 or more")
|
|
130
|
+
if (v := get("SYSTEM_METRICS")) is not None:
|
|
131
|
+
kw["system_metrics"] = _bool("SYSTEM_METRICS", v)
|
|
132
|
+
if (v := get("CAPTURE_LOGS")) is not None:
|
|
133
|
+
kw["capture_logs"] = _bool("CAPTURE_LOGS", v)
|
|
134
|
+
if (v := get("PROMOTE_METRIC")) is not None:
|
|
135
|
+
kw["promote_metric"] = v
|
|
136
|
+
if (v := get("PROMOTE_MIN")) is not None:
|
|
137
|
+
kw["promote_min"] = _float("PROMOTE_MIN", v)
|
|
138
|
+
if (v := get("PROMOTE_MUST_BEAT_PROD")) is not None:
|
|
139
|
+
kw["promote_must_beat_prod"] = _bool("PROMOTE_MUST_BEAT_PROD", v)
|
|
140
|
+
if (v := get("GATED_ALIASES")) is not None:
|
|
141
|
+
kw["gated_aliases"] = _csv(v)
|
|
142
|
+
if (v := get("OFFLINE_QUEUE")) is not None:
|
|
143
|
+
kw["offline_queue"] = _bool("OFFLINE_QUEUE", v)
|
|
144
|
+
if (v := get("QUEUE_DIR")) is not None:
|
|
145
|
+
kw["queue_dir"] = v
|
|
146
|
+
if (v := get("RETRIES")) is not None:
|
|
147
|
+
kw["retries"] = _int("RETRIES", v)
|
|
148
|
+
if (v := get("RETRY_WAIT")) is not None:
|
|
149
|
+
kw["retry_wait"] = _float("RETRY_WAIT", v)
|
|
150
|
+
return cls(**kw)
|
|
151
|
+
|
|
152
|
+
def masked(self) -> dict:
|
|
153
|
+
"""Settings as a dict with credentials in URIs hidden."""
|
|
154
|
+
d = asdict(self)
|
|
155
|
+
for k in ("tracking_uri", "registry_uri"):
|
|
156
|
+
if d[k]:
|
|
157
|
+
d[k] = re.sub(r"://[^/@]+@", "://***@", d[k])
|
|
158
|
+
return d
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
_configured: Optional[Settings] = None
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def configure(settings: Optional[Settings] = None) -> None:
|
|
165
|
+
"""Set settings explicitly (or pass None to go back to reading the environment)."""
|
|
166
|
+
global _configured
|
|
167
|
+
_configured = settings
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def get_settings() -> Settings:
|
|
171
|
+
return _configured if _configured is not None else Settings.from_env()
|