authtracker 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- authtracker/__init__.py +16 -0
- authtracker/backends/__init__.py +95 -0
- authtracker/backends/base.py +136 -0
- authtracker/backends/local.py +200 -0
- authtracker/backends/mlflow_backend.py +310 -0
- authtracker/backends/resilient.py +329 -0
- authtracker/backends/wandb_backend.py +235 -0
- authtracker/bundle.py +887 -0
- authtracker/capture.py +175 -0
- authtracker/cli.py +51 -0
- authtracker/configfile.py +84 -0
- authtracker/dataset.py +98 -0
- authtracker/errors.py +18 -0
- authtracker/registry.py +118 -0
- authtracker/run.py +346 -0
- authtracker/settings.py +171 -0
- authtracker/simple.py +241 -0
- authtracker/testing.py +82 -0
- authtracker/utils.py +321 -0
- authtracker/watch.py +59 -0
- authtracker-0.0.1.dist-info/METADATA +215 -0
- authtracker-0.0.1.dist-info/RECORD +25 -0
- authtracker-0.0.1.dist-info/WHEEL +4 -0
- authtracker-0.0.1.dist-info/entry_points.txt +2 -0
- authtracker-0.0.1.dist-info/licenses/LICENSE +202 -0
authtracker/capture.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Copy everything a run prints or logs into a file, so it can be saved as `output.log`.
|
|
2
|
+
|
|
3
|
+
While at least one capture is active, `sys.stdout` and `sys.stderr` are replaced by wrappers that
|
|
4
|
+
still write to the real streams, and also to every active capture file. Logging handlers that were
|
|
5
|
+
created before the capture started (for example by `logging.basicConfig`) hold the old stream, so
|
|
6
|
+
they are pointed at the wrapper for the duration. Everything is restored when the last capture ends.
|
|
7
|
+
|
|
8
|
+
Not captured: output of subprocesses or native code that writes straight to the terminal, and
|
|
9
|
+
loggers that write only to files.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import logging
|
|
14
|
+
import sys
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
from typing import Callable, List, Optional, Tuple
|
|
18
|
+
|
|
19
|
+
MAX_BYTES = 20 * 1024 * 1024 # stop copying after this much, so a chatty job cannot fill the disk
|
|
20
|
+
|
|
21
|
+
_lock = threading.RLock()
|
|
22
|
+
_active: List["Capture"] = []
|
|
23
|
+
_installed: dict = {}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Capture:
|
|
27
|
+
def __init__(self, path: str, periodic: Optional[Callable[[str], None]] = None,
|
|
28
|
+
interval: float = 0.0):
|
|
29
|
+
self.path = path
|
|
30
|
+
self._file = open(path, "w", encoding="utf-8", errors="replace")
|
|
31
|
+
self._size = 0
|
|
32
|
+
self._full = False
|
|
33
|
+
self.periodic = periodic # called with the file path every `interval` seconds
|
|
34
|
+
self.interval = interval
|
|
35
|
+
self.lock = threading.Lock() # held while uploading, so closing waits for an upload
|
|
36
|
+
self.closed = False
|
|
37
|
+
self._uploaded_size = -1
|
|
38
|
+
self._next_due = time.monotonic() + interval
|
|
39
|
+
|
|
40
|
+
def write(self, text: str) -> None:
|
|
41
|
+
if self._full:
|
|
42
|
+
return
|
|
43
|
+
size = len(text.encode("utf-8", "replace"))
|
|
44
|
+
if self._size + size > MAX_BYTES:
|
|
45
|
+
self._file.write(f"\n[authtracker: output.log truncated after {MAX_BYTES // 2**20} MB]\n")
|
|
46
|
+
self._full = True
|
|
47
|
+
return
|
|
48
|
+
self._file.write(text)
|
|
49
|
+
self._size += size
|
|
50
|
+
|
|
51
|
+
def tick(self) -> None:
|
|
52
|
+
"""Upload the log so far, if it grew since the last upload. Never raises."""
|
|
53
|
+
with self.lock:
|
|
54
|
+
if self.closed or self._size == self._uploaded_size:
|
|
55
|
+
return
|
|
56
|
+
try:
|
|
57
|
+
self._file.flush()
|
|
58
|
+
self.periodic(self.path)
|
|
59
|
+
self._uploaded_size = self._size
|
|
60
|
+
except Exception:
|
|
61
|
+
pass # best effort: the final upload still happens at the end
|
|
62
|
+
|
|
63
|
+
def close(self) -> None:
|
|
64
|
+
with self.lock:
|
|
65
|
+
self.closed = True
|
|
66
|
+
try:
|
|
67
|
+
self._file.close()
|
|
68
|
+
except Exception:
|
|
69
|
+
pass
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class _Tee:
|
|
73
|
+
"""Looks like the stream it wraps, and also copies what is written to the active captures."""
|
|
74
|
+
|
|
75
|
+
def __init__(self, original):
|
|
76
|
+
self._original = original
|
|
77
|
+
|
|
78
|
+
def write(self, text):
|
|
79
|
+
written = self._original.write(text)
|
|
80
|
+
with _lock:
|
|
81
|
+
for capture in _active:
|
|
82
|
+
try:
|
|
83
|
+
capture.write(text)
|
|
84
|
+
except Exception:
|
|
85
|
+
pass # capturing must never break the program
|
|
86
|
+
return written
|
|
87
|
+
|
|
88
|
+
def writelines(self, lines):
|
|
89
|
+
for line in lines:
|
|
90
|
+
self.write(line)
|
|
91
|
+
|
|
92
|
+
def flush(self):
|
|
93
|
+
self._original.flush()
|
|
94
|
+
|
|
95
|
+
def __getattr__(self, name): # isatty, encoding, fileno, ...
|
|
96
|
+
return getattr(self._original, name)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _loggers() -> List[logging.Logger]:
|
|
100
|
+
return [logging.root] + [lg for lg in logging.root.manager.loggerDict.values()
|
|
101
|
+
if isinstance(lg, logging.Logger)]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _install() -> None:
|
|
105
|
+
out, err = sys.stdout, sys.stderr
|
|
106
|
+
tee_out = _Tee(out) if out is not None else None
|
|
107
|
+
tee_err = _Tee(err) if err is not None else None
|
|
108
|
+
patched: List[Tuple[logging.StreamHandler, object]] = []
|
|
109
|
+
for lg in _loggers():
|
|
110
|
+
for h in lg.handlers:
|
|
111
|
+
if isinstance(h, logging.StreamHandler):
|
|
112
|
+
if tee_err is not None and h.stream is err:
|
|
113
|
+
patched.append((h, h.stream))
|
|
114
|
+
h.stream = tee_err
|
|
115
|
+
elif tee_out is not None and h.stream is out:
|
|
116
|
+
patched.append((h, h.stream))
|
|
117
|
+
h.stream = tee_out
|
|
118
|
+
if tee_out is not None:
|
|
119
|
+
sys.stdout = tee_out
|
|
120
|
+
if tee_err is not None:
|
|
121
|
+
sys.stderr = tee_err
|
|
122
|
+
_installed.update(out=out, err=err, tee_out=tee_out, tee_err=tee_err, patched=patched)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _uninstall() -> None:
|
|
126
|
+
if _installed.get("tee_out") is not None and sys.stdout is _installed["tee_out"]:
|
|
127
|
+
sys.stdout = _installed["out"]
|
|
128
|
+
if _installed.get("tee_err") is not None and sys.stderr is _installed["tee_err"]:
|
|
129
|
+
sys.stderr = _installed["err"]
|
|
130
|
+
for handler, stream in _installed.get("patched", []):
|
|
131
|
+
if handler.stream in (_installed["tee_out"], _installed["tee_err"]):
|
|
132
|
+
handler.stream = stream
|
|
133
|
+
_installed.clear()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
_ticker: Optional[threading.Thread] = None
|
|
137
|
+
_ticker_stop = threading.Event()
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _tick_loop() -> None:
|
|
141
|
+
while not _ticker_stop.wait(0.5):
|
|
142
|
+
with _lock:
|
|
143
|
+
due = [c for c in _active if c.periodic and c.interval > 0
|
|
144
|
+
and time.monotonic() >= c._next_due]
|
|
145
|
+
for capture in due:
|
|
146
|
+
capture._next_due = time.monotonic() + capture.interval
|
|
147
|
+
capture.tick() # outside _lock: an upload can take a while
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def start(path: str, periodic: Optional[Callable[[str], None]] = None,
|
|
151
|
+
interval: float = 0.0) -> Capture:
|
|
152
|
+
"""Start copying output into `path`. With `periodic` and `interval`, the file is also handed to
|
|
153
|
+
`periodic` every `interval` seconds while the run is going (a killed job keeps its log)."""
|
|
154
|
+
global _ticker
|
|
155
|
+
capture = Capture(path, periodic, interval)
|
|
156
|
+
with _lock:
|
|
157
|
+
if not _active:
|
|
158
|
+
_install()
|
|
159
|
+
_active.append(capture)
|
|
160
|
+
if periodic and interval > 0 and (_ticker is None or not _ticker.is_alive()):
|
|
161
|
+
_ticker_stop.clear()
|
|
162
|
+
_ticker = threading.Thread(target=_tick_loop, name="authtracker-log-upload", daemon=True)
|
|
163
|
+
_ticker.start()
|
|
164
|
+
return capture
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def stop(capture: Capture) -> None:
|
|
168
|
+
with _lock:
|
|
169
|
+
if capture in _active:
|
|
170
|
+
_active.remove(capture)
|
|
171
|
+
if not _active and _installed:
|
|
172
|
+
_uninstall()
|
|
173
|
+
if not _active:
|
|
174
|
+
_ticker_stop.set()
|
|
175
|
+
capture.close() # waits for an upload that is in progress
|
authtracker/cli.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""authtracker config show | doctor"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from .backends import get_backend
|
|
9
|
+
from .settings import get_settings
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def main(argv=None) -> int:
|
|
13
|
+
p = argparse.ArgumentParser(prog="authtracker")
|
|
14
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
15
|
+
cfg = sub.add_parser("config", help="show resolved settings")
|
|
16
|
+
cfg.add_argument("action", choices=["show"])
|
|
17
|
+
sub.add_parser("doctor", help="check settings and backend connectivity")
|
|
18
|
+
sub.add_parser("sync", help="upload operations queued while the server was unreachable")
|
|
19
|
+
args = p.parse_args(argv)
|
|
20
|
+
|
|
21
|
+
s = get_settings()
|
|
22
|
+
if args.cmd == "config":
|
|
23
|
+
print(json.dumps(s.masked(), indent=2))
|
|
24
|
+
return 0
|
|
25
|
+
|
|
26
|
+
if args.cmd == "sync":
|
|
27
|
+
b = get_backend(s)
|
|
28
|
+
if not hasattr(b, "flush"):
|
|
29
|
+
print("Offline queue is disabled (AUTHTRACKER_OFFLINE_QUEUE=false)")
|
|
30
|
+
return 0
|
|
31
|
+
before = b.pending()
|
|
32
|
+
done = b.flush(force=True, all_tokens=True)
|
|
33
|
+
print(f"uploaded {done} of {before} queued operations; {b.pending()} remain")
|
|
34
|
+
return 0 if b.pending() == 0 else 1
|
|
35
|
+
|
|
36
|
+
print(f"backend : {s.backend}")
|
|
37
|
+
print(f"tracking_uri : {s.masked()['tracking_uri']}")
|
|
38
|
+
try:
|
|
39
|
+
get_backend(s).ping()
|
|
40
|
+
except Exception as e:
|
|
41
|
+
print(f"FAIL: {type(e).__name__}: {e}")
|
|
42
|
+
return 1
|
|
43
|
+
print("OK: backend reachable")
|
|
44
|
+
b = get_backend(s)
|
|
45
|
+
if hasattr(b, "pending"):
|
|
46
|
+
print(f"queued ops : {b.pending()} (authtracker sync uploads them)")
|
|
47
|
+
return 0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
if __name__ == "__main__":
|
|
51
|
+
sys.exit(main())
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Read a training YAML file.
|
|
2
|
+
|
|
3
|
+
authtracker: # optional block: where this run belongs
|
|
4
|
+
model: fake-image
|
|
5
|
+
experiment: baseline
|
|
6
|
+
dataset: {name: faces, version: v3, path: /data/faces}
|
|
7
|
+
tags: {team: vision}
|
|
8
|
+
optimizer: {name: adam, lr: 0.001} # everything else becomes parameters of the run
|
|
9
|
+
epochs: 15
|
|
10
|
+
|
|
11
|
+
Everything outside the `authtracker:` block is logged as parameters (nested keys are joined with
|
|
12
|
+
dots: `optimizer.lr`). Arguments passed to `authtracker.init(...)` override the block.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
from typing import Any, Dict, Iterator, Mapping, Tuple
|
|
19
|
+
|
|
20
|
+
from .errors import ValidationError
|
|
21
|
+
|
|
22
|
+
BLOCK = "authtracker"
|
|
23
|
+
BLOCK_KEYS = {"model", "experiment", "dataset", "dataset_version", "tags", "owner"}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _max_param_length() -> int:
|
|
27
|
+
try:
|
|
28
|
+
from mlflow.utils.validation import MAX_PARAM_VAL_LENGTH
|
|
29
|
+
return int(MAX_PARAM_VAL_LENGTH)
|
|
30
|
+
except Exception:
|
|
31
|
+
return 500
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _scalar(value: Any) -> Any:
|
|
35
|
+
if isinstance(value, (bool, int, float, str)):
|
|
36
|
+
text = value
|
|
37
|
+
elif value is None:
|
|
38
|
+
text = "null"
|
|
39
|
+
else: # lists and anything else: readable JSON
|
|
40
|
+
text = json.dumps(value, default=str)
|
|
41
|
+
limit = _max_param_length()
|
|
42
|
+
if isinstance(text, str) and len(text) > limit:
|
|
43
|
+
text = text[: limit - 20] + "...(truncated)"
|
|
44
|
+
return text
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def flatten(data: Mapping[str, Any], prefix: str = "") -> Iterator[Tuple[str, Any]]:
|
|
48
|
+
"""{'a': {'b': 1}} -> ('a.b', 1). Lists become JSON text."""
|
|
49
|
+
for key, value in data.items():
|
|
50
|
+
name = f"{prefix}{key}"
|
|
51
|
+
if isinstance(value, Mapping):
|
|
52
|
+
yield from flatten(value, name + ".")
|
|
53
|
+
else:
|
|
54
|
+
yield name, _scalar(value)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def flat_params(data: Mapping[str, Any]) -> Dict[str, Any]:
|
|
58
|
+
return dict(flatten(data))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def load(path) -> Tuple[Dict[str, Any], Dict[str, Any], str]:
|
|
62
|
+
"""Return (params, tracking block, absolute path) for the YAML file at `path`."""
|
|
63
|
+
path = os.fspath(path)
|
|
64
|
+
if not os.path.isfile(path):
|
|
65
|
+
raise ValidationError(f"Training config {path!r} does not exist")
|
|
66
|
+
try:
|
|
67
|
+
import yaml
|
|
68
|
+
except ImportError as e: # pragma: no cover
|
|
69
|
+
raise ValidationError("Reading a YAML config needs PyYAML: pip install pyyaml") from e
|
|
70
|
+
with open(path, encoding="utf-8") as f:
|
|
71
|
+
try:
|
|
72
|
+
data = yaml.safe_load(f) or {}
|
|
73
|
+
except yaml.YAMLError as e:
|
|
74
|
+
raise ValidationError(f"{path!r} is not valid YAML: {e}") from e
|
|
75
|
+
if not isinstance(data, dict):
|
|
76
|
+
raise ValidationError(f"{path!r} must contain a mapping at the top level")
|
|
77
|
+
block = data.pop(BLOCK, None) or {}
|
|
78
|
+
if not isinstance(block, dict):
|
|
79
|
+
raise ValidationError(f"'{BLOCK}:' in {path!r} must be a mapping")
|
|
80
|
+
unknown = sorted(set(block) - BLOCK_KEYS)
|
|
81
|
+
if unknown:
|
|
82
|
+
raise ValidationError(
|
|
83
|
+
f"Unknown keys {unknown} in the '{BLOCK}:' block of {path!r}; allowed: {sorted(BLOCK_KEYS)}")
|
|
84
|
+
return flat_params(data), block, os.path.abspath(path)
|
authtracker/dataset.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Describe the data a run used, with a fingerprint (hash) of it.
|
|
2
|
+
|
|
3
|
+
dataset={"name": "faces", "version": "v3", "path": "/data/faces"}
|
|
4
|
+
dataset={"name": "faces", "version": "v3", "path": "/data/faces", "hash": "contents"}
|
|
5
|
+
dataset="/data/faces" # just a path
|
|
6
|
+
|
|
7
|
+
Hash modes:
|
|
8
|
+
manifest (default) fingerprint of the list of files: relative path and size of each. Fast even
|
|
9
|
+
for millions of files, and it catches files that were added, removed,
|
|
10
|
+
renamed or changed size. It does not read the files.
|
|
11
|
+
contents also reads every byte (slow on big data); catches a changed file of the same size.
|
|
12
|
+
none no hash.
|
|
13
|
+
|
|
14
|
+
Only local paths can be hashed. For a remote path (s3://, gs://, ...) the path is recorded and no
|
|
15
|
+
hash is computed.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import hashlib
|
|
20
|
+
import os
|
|
21
|
+
from typing import Any, Dict, Mapping, Optional, Union
|
|
22
|
+
|
|
23
|
+
from .errors import ValidationError
|
|
24
|
+
|
|
25
|
+
MODES = ("manifest", "contents", "none")
|
|
26
|
+
_CHUNK = 1024 * 1024
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _files(path: str):
|
|
30
|
+
"""Yield (relative path with '/', absolute path, size) in a stable order."""
|
|
31
|
+
if os.path.isfile(path):
|
|
32
|
+
yield os.path.basename(path), path, os.path.getsize(path)
|
|
33
|
+
return
|
|
34
|
+
entries = []
|
|
35
|
+
for root, dirs, names in os.walk(path):
|
|
36
|
+
dirs.sort()
|
|
37
|
+
for name in sorted(names):
|
|
38
|
+
full = os.path.join(root, name)
|
|
39
|
+
if os.path.isfile(full):
|
|
40
|
+
entries.append((os.path.relpath(full, path).replace(os.sep, "/"), full, os.path.getsize(full)))
|
|
41
|
+
yield from sorted(entries)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def fingerprint(path: str, mode: str = "manifest") -> Dict[str, Any]:
|
|
45
|
+
"""Hash a local file or folder. Returns {'hash', 'files', 'bytes'}."""
|
|
46
|
+
h = hashlib.sha256()
|
|
47
|
+
count = total = 0
|
|
48
|
+
for rel, full, size in _files(path):
|
|
49
|
+
h.update(f"{rel}\t{size}\n".encode())
|
|
50
|
+
if mode == "contents":
|
|
51
|
+
with open(full, "rb") as f:
|
|
52
|
+
for chunk in iter(lambda: f.read(_CHUNK), b""):
|
|
53
|
+
h.update(chunk)
|
|
54
|
+
count += 1
|
|
55
|
+
total += size
|
|
56
|
+
return {"hash": h.hexdigest(), "files": count, "bytes": total}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def describe(spec: Union[str, os.PathLike, Mapping[str, Any]]) -> Dict[str, Any]:
|
|
60
|
+
"""Turn the `dataset=` argument into a record: name, version, path, hash, hash_mode, files, bytes."""
|
|
61
|
+
if isinstance(spec, (str, os.PathLike)):
|
|
62
|
+
spec = {"path": os.fspath(spec)}
|
|
63
|
+
if not isinstance(spec, Mapping):
|
|
64
|
+
raise ValidationError("dataset must be a path or a dict with name / version / path")
|
|
65
|
+
allowed = {"name", "version", "path", "hash"}
|
|
66
|
+
unknown = sorted(set(spec) - allowed)
|
|
67
|
+
if unknown:
|
|
68
|
+
raise ValidationError(f"Unknown dataset keys {unknown}; allowed: {sorted(allowed)}")
|
|
69
|
+
|
|
70
|
+
path: Optional[str] = os.fspath(spec["path"]) if spec.get("path") else None
|
|
71
|
+
mode = spec.get("hash", "manifest")
|
|
72
|
+
if mode not in MODES:
|
|
73
|
+
raise ValidationError(f"dataset hash must be one of {list(MODES)}, got {mode!r}")
|
|
74
|
+
name = spec.get("name") or (os.path.basename(path.rstrip("/")) if path else None)
|
|
75
|
+
if not name and not path:
|
|
76
|
+
raise ValidationError("dataset needs at least a name or a path")
|
|
77
|
+
|
|
78
|
+
record: Dict[str, Any] = {"name": name, "version": spec.get("version"), "path": path,
|
|
79
|
+
"hash": None, "hash_mode": "none", "files": None, "bytes": None}
|
|
80
|
+
if path and mode != "none":
|
|
81
|
+
if "://" in path:
|
|
82
|
+
record["hash_mode"] = "remote path: not hashed"
|
|
83
|
+
elif not os.path.exists(path):
|
|
84
|
+
raise ValidationError(f"dataset path {path!r} does not exist")
|
|
85
|
+
else:
|
|
86
|
+
record.update(fingerprint(path, mode))
|
|
87
|
+
record["hash_mode"] = mode
|
|
88
|
+
return record
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def as_tags(record: Mapping[str, Any]) -> Dict[str, str]:
|
|
92
|
+
"""The record as run tags, so you can search runs by dataset in the UI."""
|
|
93
|
+
keys = {"name": "dataset_name", "path": "dataset_path", "hash": "dataset_hash",
|
|
94
|
+
"files": "dataset_files", "bytes": "dataset_bytes", "hash_mode": "dataset_hash_mode"}
|
|
95
|
+
tags = {tag: str(record[k]) for k, tag in keys.items() if record.get(k) is not None}
|
|
96
|
+
if record.get("hash"):
|
|
97
|
+
tags["dataset_hash"] = record["hash"][:16] # short form: readable in the UI
|
|
98
|
+
return tags
|
authtracker/errors.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
class AuthTrackerError(Exception):
|
|
2
|
+
"""Base class for authtracker errors."""
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class ValidationError(AuthTrackerError, ValueError):
|
|
6
|
+
"""A run violates the configured convention (tags, model name, metric names)."""
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class PromotionError(AuthTrackerError):
|
|
10
|
+
"""A model version failed a promotion gate."""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class BackendError(AuthTrackerError):
|
|
14
|
+
"""The backend is unknown, missing a dependency, or does not support an operation."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class UnsupportedError(BackendError):
|
|
18
|
+
"""The backend does not implement this optional feature (for example resuming runs)."""
|
authtracker/registry.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Versioning: register, promote (with gates), rollback, load, best_run."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import shutil
|
|
5
|
+
from typing import Any, Dict, Optional, Union
|
|
6
|
+
|
|
7
|
+
from . import bundle
|
|
8
|
+
from .backends import Backend, get_backend
|
|
9
|
+
from .backends.base import TAG_LEVEL, TAG_MODEL_URI
|
|
10
|
+
from .errors import PromotionError, ValidationError
|
|
11
|
+
from .run import Run
|
|
12
|
+
from .settings import Settings, get_settings
|
|
13
|
+
|
|
14
|
+
RunRef = Union[Run, str, Dict[str, Any]]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _ctx(settings: Optional[Settings], backend: Optional[Backend]):
|
|
18
|
+
s = settings or get_settings()
|
|
19
|
+
return s, backend or get_backend(s)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _run_id(ref: RunRef) -> str:
|
|
23
|
+
if isinstance(ref, Run):
|
|
24
|
+
return ref.id
|
|
25
|
+
if isinstance(ref, dict):
|
|
26
|
+
return ref["run_id"]
|
|
27
|
+
return ref
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def best_run(model: str, experiment: str, metric: str, mode: str = "max",
|
|
31
|
+
*, settings=None, backend=None) -> Optional[Dict[str, Any]]:
|
|
32
|
+
"""Best child run of `experiment` under `model` by `metric` (None if no runs)."""
|
|
33
|
+
if mode not in ("max", "min"):
|
|
34
|
+
raise ValueError("mode must be 'max' or 'min'")
|
|
35
|
+
_, b = _ctx(settings, backend)
|
|
36
|
+
runs = b.search_runs(model, {"model": model, "experiment": experiment, TAG_LEVEL: "run"},
|
|
37
|
+
metric, mode, 1)
|
|
38
|
+
return runs[0] if runs else None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def register(run: Optional[RunRef] = None, name: Optional[str] = None, *,
|
|
42
|
+
settings=None, backend=None) -> str:
|
|
43
|
+
"""Create a new model version from the model logged in `run` (default: the current run)."""
|
|
44
|
+
_, b = _ctx(settings, backend)
|
|
45
|
+
if run is None:
|
|
46
|
+
from .simple import current_run
|
|
47
|
+
run = current_run()
|
|
48
|
+
info = b.get_run(_run_id(run))
|
|
49
|
+
uri = info["tags"].get(TAG_MODEL_URI)
|
|
50
|
+
if not uri:
|
|
51
|
+
raise ValidationError("This run has no logged model; call run.log_model(...) first")
|
|
52
|
+
return b.register(info["run_id"], name or info["tags"]["model"], uri)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _metric(b: Backend, name: str, version: str, metric: str) -> Optional[float]:
|
|
56
|
+
return b.get_run(b.version_run_id(name, version))["metrics"].get(metric)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _check_gate(s: Settings, b: Backend, name: str, version: str, to: str) -> None:
|
|
60
|
+
if s.promote_metric is None:
|
|
61
|
+
return
|
|
62
|
+
value = _metric(b, name, version, s.promote_metric)
|
|
63
|
+
if value is None:
|
|
64
|
+
raise PromotionError(f"Version {version} has no metric {s.promote_metric!r}")
|
|
65
|
+
if s.promote_min is not None and value < s.promote_min:
|
|
66
|
+
raise PromotionError(
|
|
67
|
+
f"{s.promote_metric}={value} is below the minimum {s.promote_min} for {to!r}")
|
|
68
|
+
if s.promote_must_beat_prod:
|
|
69
|
+
current = b.get_alias(name, to)
|
|
70
|
+
if current is not None and current != version:
|
|
71
|
+
cur_value = _metric(b, name, current, s.promote_metric)
|
|
72
|
+
if cur_value is not None and value <= cur_value:
|
|
73
|
+
raise PromotionError(
|
|
74
|
+
f"{s.promote_metric}={value} does not beat current {to!r} "
|
|
75
|
+
f"(v{current}: {cur_value})")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def promote(name: str, version: Union[str, int], to: str, *, settings=None, backend=None) -> None:
|
|
79
|
+
"""Point alias `to` at `version`. Gated aliases must pass the configured checks."""
|
|
80
|
+
s, b = _ctx(settings, backend)
|
|
81
|
+
version = str(version)
|
|
82
|
+
b.version_run_id(name, version) # raises if it does not exist
|
|
83
|
+
if to in s.gated_aliases:
|
|
84
|
+
_check_gate(s, b, name, version, to)
|
|
85
|
+
previous = b.get_alias(name, to)
|
|
86
|
+
if previous is not None and previous != version:
|
|
87
|
+
b.set_alias(name, previous, f"previous-{to}")
|
|
88
|
+
b.set_alias(name, version, to)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def rollback(name: str, alias: str = "production", *, settings=None, backend=None) -> str:
|
|
92
|
+
"""Move `alias` back to the version it pointed at before the last promote."""
|
|
93
|
+
_, b = _ctx(settings, backend)
|
|
94
|
+
previous = b.get_alias(name, f"previous-{alias}")
|
|
95
|
+
if previous is None:
|
|
96
|
+
raise PromotionError(f"No previous version recorded for alias {alias!r}")
|
|
97
|
+
current = b.get_alias(name, alias)
|
|
98
|
+
b.set_alias(name, previous, alias)
|
|
99
|
+
if current is not None:
|
|
100
|
+
b.set_alias(name, current, f"previous-{alias}")
|
|
101
|
+
return previous
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def load(name: str, alias: str = "production", *, device: str = "cpu", pickle: bool = False,
|
|
105
|
+
settings=None, backend=None) -> Any:
|
|
106
|
+
"""Load a model by registry alias (never by run id).
|
|
107
|
+
|
|
108
|
+
A PyTorch bundle comes back as the real torch.nn.Module (on `device`, in eval mode); any other
|
|
109
|
+
model comes back as an MLflow model with predict().
|
|
110
|
+
"""
|
|
111
|
+
_, b = _ctx(settings, backend)
|
|
112
|
+
folder = b.download_bundle(name, alias)
|
|
113
|
+
if folder is None:
|
|
114
|
+
return b.load(name, alias)
|
|
115
|
+
try:
|
|
116
|
+
return bundle.load_dir(folder, device=device, pickle=pickle)
|
|
117
|
+
finally:
|
|
118
|
+
shutil.rmtree(folder, ignore_errors=True)
|