authtracker 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
authtracker/capture.py ADDED
@@ -0,0 +1,175 @@
1
+ """Copy everything a run prints or logs into a file, so it can be saved as `output.log`.
2
+
3
+ While at least one capture is active, `sys.stdout` and `sys.stderr` are replaced by wrappers that
4
+ still write to the real streams, and also to every active capture file. Logging handlers that were
5
+ created before the capture started (for example by `logging.basicConfig`) hold the old stream, so
6
+ they are pointed at the wrapper for the duration. Everything is restored when the last capture ends.
7
+
8
+ Not captured: output of subprocesses or native code that writes straight to the terminal, and
9
+ loggers that write only to files.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import logging
14
+ import sys
15
+ import threading
16
+ import time
17
+ from typing import Callable, List, Optional, Tuple
18
+
19
+ MAX_BYTES = 20 * 1024 * 1024 # stop copying after this much, so a chatty job cannot fill the disk
20
+
21
+ _lock = threading.RLock()
22
+ _active: List["Capture"] = []
23
+ _installed: dict = {}
24
+
25
+
26
+ class Capture:
27
+ def __init__(self, path: str, periodic: Optional[Callable[[str], None]] = None,
28
+ interval: float = 0.0):
29
+ self.path = path
30
+ self._file = open(path, "w", encoding="utf-8", errors="replace")
31
+ self._size = 0
32
+ self._full = False
33
+ self.periodic = periodic # called with the file path every `interval` seconds
34
+ self.interval = interval
35
+ self.lock = threading.Lock() # held while uploading, so closing waits for an upload
36
+ self.closed = False
37
+ self._uploaded_size = -1
38
+ self._next_due = time.monotonic() + interval
39
+
40
+ def write(self, text: str) -> None:
41
+ if self._full:
42
+ return
43
+ size = len(text.encode("utf-8", "replace"))
44
+ if self._size + size > MAX_BYTES:
45
+ self._file.write(f"\n[authtracker: output.log truncated after {MAX_BYTES // 2**20} MB]\n")
46
+ self._full = True
47
+ return
48
+ self._file.write(text)
49
+ self._size += size
50
+
51
+ def tick(self) -> None:
52
+ """Upload the log so far, if it grew since the last upload. Never raises."""
53
+ with self.lock:
54
+ if self.closed or self._size == self._uploaded_size:
55
+ return
56
+ try:
57
+ self._file.flush()
58
+ self.periodic(self.path)
59
+ self._uploaded_size = self._size
60
+ except Exception:
61
+ pass # best effort: the final upload still happens at the end
62
+
63
+ def close(self) -> None:
64
+ with self.lock:
65
+ self.closed = True
66
+ try:
67
+ self._file.close()
68
+ except Exception:
69
+ pass
70
+
71
+
72
+ class _Tee:
73
+ """Looks like the stream it wraps, and also copies what is written to the active captures."""
74
+
75
+ def __init__(self, original):
76
+ self._original = original
77
+
78
+ def write(self, text):
79
+ written = self._original.write(text)
80
+ with _lock:
81
+ for capture in _active:
82
+ try:
83
+ capture.write(text)
84
+ except Exception:
85
+ pass # capturing must never break the program
86
+ return written
87
+
88
+ def writelines(self, lines):
89
+ for line in lines:
90
+ self.write(line)
91
+
92
+ def flush(self):
93
+ self._original.flush()
94
+
95
+ def __getattr__(self, name): # isatty, encoding, fileno, ...
96
+ return getattr(self._original, name)
97
+
98
+
99
+ def _loggers() -> List[logging.Logger]:
100
+ return [logging.root] + [lg for lg in logging.root.manager.loggerDict.values()
101
+ if isinstance(lg, logging.Logger)]
102
+
103
+
104
+ def _install() -> None:
105
+ out, err = sys.stdout, sys.stderr
106
+ tee_out = _Tee(out) if out is not None else None
107
+ tee_err = _Tee(err) if err is not None else None
108
+ patched: List[Tuple[logging.StreamHandler, object]] = []
109
+ for lg in _loggers():
110
+ for h in lg.handlers:
111
+ if isinstance(h, logging.StreamHandler):
112
+ if tee_err is not None and h.stream is err:
113
+ patched.append((h, h.stream))
114
+ h.stream = tee_err
115
+ elif tee_out is not None and h.stream is out:
116
+ patched.append((h, h.stream))
117
+ h.stream = tee_out
118
+ if tee_out is not None:
119
+ sys.stdout = tee_out
120
+ if tee_err is not None:
121
+ sys.stderr = tee_err
122
+ _installed.update(out=out, err=err, tee_out=tee_out, tee_err=tee_err, patched=patched)
123
+
124
+
125
+ def _uninstall() -> None:
126
+ if _installed.get("tee_out") is not None and sys.stdout is _installed["tee_out"]:
127
+ sys.stdout = _installed["out"]
128
+ if _installed.get("tee_err") is not None and sys.stderr is _installed["tee_err"]:
129
+ sys.stderr = _installed["err"]
130
+ for handler, stream in _installed.get("patched", []):
131
+ if handler.stream in (_installed["tee_out"], _installed["tee_err"]):
132
+ handler.stream = stream
133
+ _installed.clear()
134
+
135
+
136
+ _ticker: Optional[threading.Thread] = None
137
+ _ticker_stop = threading.Event()
138
+
139
+
140
+ def _tick_loop() -> None:
141
+ while not _ticker_stop.wait(0.5):
142
+ with _lock:
143
+ due = [c for c in _active if c.periodic and c.interval > 0
144
+ and time.monotonic() >= c._next_due]
145
+ for capture in due:
146
+ capture._next_due = time.monotonic() + capture.interval
147
+ capture.tick() # outside _lock: an upload can take a while
148
+
149
+
150
+ def start(path: str, periodic: Optional[Callable[[str], None]] = None,
151
+ interval: float = 0.0) -> Capture:
152
+ """Start copying output into `path`. With `periodic` and `interval`, the file is also handed to
153
+ `periodic` every `interval` seconds while the run is going (a killed job keeps its log)."""
154
+ global _ticker
155
+ capture = Capture(path, periodic, interval)
156
+ with _lock:
157
+ if not _active:
158
+ _install()
159
+ _active.append(capture)
160
+ if periodic and interval > 0 and (_ticker is None or not _ticker.is_alive()):
161
+ _ticker_stop.clear()
162
+ _ticker = threading.Thread(target=_tick_loop, name="authtracker-log-upload", daemon=True)
163
+ _ticker.start()
164
+ return capture
165
+
166
+
167
+ def stop(capture: Capture) -> None:
168
+ with _lock:
169
+ if capture in _active:
170
+ _active.remove(capture)
171
+ if not _active and _installed:
172
+ _uninstall()
173
+ if not _active:
174
+ _ticker_stop.set()
175
+ capture.close() # waits for an upload that is in progress
authtracker/cli.py ADDED
@@ -0,0 +1,51 @@
1
+ """authtracker config show | doctor"""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import sys
7
+
8
+ from .backends import get_backend
9
+ from .settings import get_settings
10
+
11
+
12
+ def main(argv=None) -> int:
13
+ p = argparse.ArgumentParser(prog="authtracker")
14
+ sub = p.add_subparsers(dest="cmd", required=True)
15
+ cfg = sub.add_parser("config", help="show resolved settings")
16
+ cfg.add_argument("action", choices=["show"])
17
+ sub.add_parser("doctor", help="check settings and backend connectivity")
18
+ sub.add_parser("sync", help="upload operations queued while the server was unreachable")
19
+ args = p.parse_args(argv)
20
+
21
+ s = get_settings()
22
+ if args.cmd == "config":
23
+ print(json.dumps(s.masked(), indent=2))
24
+ return 0
25
+
26
+ if args.cmd == "sync":
27
+ b = get_backend(s)
28
+ if not hasattr(b, "flush"):
29
+ print("Offline queue is disabled (AUTHTRACKER_OFFLINE_QUEUE=false)")
30
+ return 0
31
+ before = b.pending()
32
+ done = b.flush(force=True, all_tokens=True)
33
+ print(f"uploaded {done} of {before} queued operations; {b.pending()} remain")
34
+ return 0 if b.pending() == 0 else 1
35
+
36
+ print(f"backend : {s.backend}")
37
+ print(f"tracking_uri : {s.masked()['tracking_uri']}")
38
+ try:
39
+ get_backend(s).ping()
40
+ except Exception as e:
41
+ print(f"FAIL: {type(e).__name__}: {e}")
42
+ return 1
43
+ print("OK: backend reachable")
44
+ b = get_backend(s)
45
+ if hasattr(b, "pending"):
46
+ print(f"queued ops : {b.pending()} (authtracker sync uploads them)")
47
+ return 0
48
+
49
+
50
+ if __name__ == "__main__":
51
+ sys.exit(main())
@@ -0,0 +1,84 @@
1
+ """Read a training YAML file.
2
+
3
+ authtracker: # optional block: where this run belongs
4
+ model: fake-image
5
+ experiment: baseline
6
+ dataset: {name: faces, version: v3, path: /data/faces}
7
+ tags: {team: vision}
8
+ optimizer: {name: adam, lr: 0.001} # everything else becomes parameters of the run
9
+ epochs: 15
10
+
11
+ Everything outside the `authtracker:` block is logged as parameters (nested keys are joined with
12
+ dots: `optimizer.lr`). Arguments passed to `authtracker.init(...)` override the block.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ import os
18
+ from typing import Any, Dict, Iterator, Mapping, Tuple
19
+
20
+ from .errors import ValidationError
21
+
22
+ BLOCK = "authtracker"
23
+ BLOCK_KEYS = {"model", "experiment", "dataset", "dataset_version", "tags", "owner"}
24
+
25
+
26
+ def _max_param_length() -> int:
27
+ try:
28
+ from mlflow.utils.validation import MAX_PARAM_VAL_LENGTH
29
+ return int(MAX_PARAM_VAL_LENGTH)
30
+ except Exception:
31
+ return 500
32
+
33
+
34
+ def _scalar(value: Any) -> Any:
35
+ if isinstance(value, (bool, int, float, str)):
36
+ text = value
37
+ elif value is None:
38
+ text = "null"
39
+ else: # lists and anything else: readable JSON
40
+ text = json.dumps(value, default=str)
41
+ limit = _max_param_length()
42
+ if isinstance(text, str) and len(text) > limit:
43
+ text = text[: limit - 20] + "...(truncated)"
44
+ return text
45
+
46
+
47
+ def flatten(data: Mapping[str, Any], prefix: str = "") -> Iterator[Tuple[str, Any]]:
48
+ """{'a': {'b': 1}} -> ('a.b', 1). Lists become JSON text."""
49
+ for key, value in data.items():
50
+ name = f"{prefix}{key}"
51
+ if isinstance(value, Mapping):
52
+ yield from flatten(value, name + ".")
53
+ else:
54
+ yield name, _scalar(value)
55
+
56
+
57
+ def flat_params(data: Mapping[str, Any]) -> Dict[str, Any]:
58
+ return dict(flatten(data))
59
+
60
+
61
+ def load(path) -> Tuple[Dict[str, Any], Dict[str, Any], str]:
62
+ """Return (params, tracking block, absolute path) for the YAML file at `path`."""
63
+ path = os.fspath(path)
64
+ if not os.path.isfile(path):
65
+ raise ValidationError(f"Training config {path!r} does not exist")
66
+ try:
67
+ import yaml
68
+ except ImportError as e: # pragma: no cover
69
+ raise ValidationError("Reading a YAML config needs PyYAML: pip install pyyaml") from e
70
+ with open(path, encoding="utf-8") as f:
71
+ try:
72
+ data = yaml.safe_load(f) or {}
73
+ except yaml.YAMLError as e:
74
+ raise ValidationError(f"{path!r} is not valid YAML: {e}") from e
75
+ if not isinstance(data, dict):
76
+ raise ValidationError(f"{path!r} must contain a mapping at the top level")
77
+ block = data.pop(BLOCK, None) or {}
78
+ if not isinstance(block, dict):
79
+ raise ValidationError(f"'{BLOCK}:' in {path!r} must be a mapping")
80
+ unknown = sorted(set(block) - BLOCK_KEYS)
81
+ if unknown:
82
+ raise ValidationError(
83
+ f"Unknown keys {unknown} in the '{BLOCK}:' block of {path!r}; allowed: {sorted(BLOCK_KEYS)}")
84
+ return flat_params(data), block, os.path.abspath(path)
authtracker/dataset.py ADDED
@@ -0,0 +1,98 @@
1
+ """Describe the data a run used, with a fingerprint (hash) of it.
2
+
3
+ dataset={"name": "faces", "version": "v3", "path": "/data/faces"}
4
+ dataset={"name": "faces", "version": "v3", "path": "/data/faces", "hash": "contents"}
5
+ dataset="/data/faces" # just a path
6
+
7
+ Hash modes:
8
+ manifest (default) fingerprint of the list of files: relative path and size of each. Fast even
9
+ for millions of files, and it catches files that were added, removed,
10
+ renamed or changed size. It does not read the files.
11
+ contents also reads every byte (slow on big data); catches a changed file of the same size.
12
+ none no hash.
13
+
14
+ Only local paths can be hashed. For a remote path (s3://, gs://, ...) the path is recorded and no
15
+ hash is computed.
16
+ """
17
+ from __future__ import annotations
18
+
19
+ import hashlib
20
+ import os
21
+ from typing import Any, Dict, Mapping, Optional, Union
22
+
23
+ from .errors import ValidationError
24
+
25
+ MODES = ("manifest", "contents", "none")
26
+ _CHUNK = 1024 * 1024
27
+
28
+
29
+ def _files(path: str):
30
+ """Yield (relative path with '/', absolute path, size) in a stable order."""
31
+ if os.path.isfile(path):
32
+ yield os.path.basename(path), path, os.path.getsize(path)
33
+ return
34
+ entries = []
35
+ for root, dirs, names in os.walk(path):
36
+ dirs.sort()
37
+ for name in sorted(names):
38
+ full = os.path.join(root, name)
39
+ if os.path.isfile(full):
40
+ entries.append((os.path.relpath(full, path).replace(os.sep, "/"), full, os.path.getsize(full)))
41
+ yield from sorted(entries)
42
+
43
+
44
+ def fingerprint(path: str, mode: str = "manifest") -> Dict[str, Any]:
45
+ """Hash a local file or folder. Returns {'hash', 'files', 'bytes'}."""
46
+ h = hashlib.sha256()
47
+ count = total = 0
48
+ for rel, full, size in _files(path):
49
+ h.update(f"{rel}\t{size}\n".encode())
50
+ if mode == "contents":
51
+ with open(full, "rb") as f:
52
+ for chunk in iter(lambda: f.read(_CHUNK), b""):
53
+ h.update(chunk)
54
+ count += 1
55
+ total += size
56
+ return {"hash": h.hexdigest(), "files": count, "bytes": total}
57
+
58
+
59
+ def describe(spec: Union[str, os.PathLike, Mapping[str, Any]]) -> Dict[str, Any]:
60
+ """Turn the `dataset=` argument into a record: name, version, path, hash, hash_mode, files, bytes."""
61
+ if isinstance(spec, (str, os.PathLike)):
62
+ spec = {"path": os.fspath(spec)}
63
+ if not isinstance(spec, Mapping):
64
+ raise ValidationError("dataset must be a path or a dict with name / version / path")
65
+ allowed = {"name", "version", "path", "hash"}
66
+ unknown = sorted(set(spec) - allowed)
67
+ if unknown:
68
+ raise ValidationError(f"Unknown dataset keys {unknown}; allowed: {sorted(allowed)}")
69
+
70
+ path: Optional[str] = os.fspath(spec["path"]) if spec.get("path") else None
71
+ mode = spec.get("hash", "manifest")
72
+ if mode not in MODES:
73
+ raise ValidationError(f"dataset hash must be one of {list(MODES)}, got {mode!r}")
74
+ name = spec.get("name") or (os.path.basename(path.rstrip("/")) if path else None)
75
+ if not name and not path:
76
+ raise ValidationError("dataset needs at least a name or a path")
77
+
78
+ record: Dict[str, Any] = {"name": name, "version": spec.get("version"), "path": path,
79
+ "hash": None, "hash_mode": "none", "files": None, "bytes": None}
80
+ if path and mode != "none":
81
+ if "://" in path:
82
+ record["hash_mode"] = "remote path: not hashed"
83
+ elif not os.path.exists(path):
84
+ raise ValidationError(f"dataset path {path!r} does not exist")
85
+ else:
86
+ record.update(fingerprint(path, mode))
87
+ record["hash_mode"] = mode
88
+ return record
89
+
90
+
91
+ def as_tags(record: Mapping[str, Any]) -> Dict[str, str]:
92
+ """The record as run tags, so you can search runs by dataset in the UI."""
93
+ keys = {"name": "dataset_name", "path": "dataset_path", "hash": "dataset_hash",
94
+ "files": "dataset_files", "bytes": "dataset_bytes", "hash_mode": "dataset_hash_mode"}
95
+ tags = {tag: str(record[k]) for k, tag in keys.items() if record.get(k) is not None}
96
+ if record.get("hash"):
97
+ tags["dataset_hash"] = record["hash"][:16] # short form: readable in the UI
98
+ return tags
authtracker/errors.py ADDED
@@ -0,0 +1,18 @@
1
+ class AuthTrackerError(Exception):
2
+ """Base class for authtracker errors."""
3
+
4
+
5
+ class ValidationError(AuthTrackerError, ValueError):
6
+ """A run violates the configured convention (tags, model name, metric names)."""
7
+
8
+
9
+ class PromotionError(AuthTrackerError):
10
+ """A model version failed a promotion gate."""
11
+
12
+
13
+ class BackendError(AuthTrackerError):
14
+ """The backend is unknown, missing a dependency, or does not support an operation."""
15
+
16
+
17
+ class UnsupportedError(BackendError):
18
+ """The backend does not implement this optional feature (for example resuming runs)."""
@@ -0,0 +1,118 @@
1
+ """Versioning: register, promote (with gates), rollback, load, best_run."""
2
+ from __future__ import annotations
3
+
4
+ import shutil
5
+ from typing import Any, Dict, Optional, Union
6
+
7
+ from . import bundle
8
+ from .backends import Backend, get_backend
9
+ from .backends.base import TAG_LEVEL, TAG_MODEL_URI
10
+ from .errors import PromotionError, ValidationError
11
+ from .run import Run
12
+ from .settings import Settings, get_settings
13
+
14
+ RunRef = Union[Run, str, Dict[str, Any]]
15
+
16
+
17
+ def _ctx(settings: Optional[Settings], backend: Optional[Backend]):
18
+ s = settings or get_settings()
19
+ return s, backend or get_backend(s)
20
+
21
+
22
+ def _run_id(ref: RunRef) -> str:
23
+ if isinstance(ref, Run):
24
+ return ref.id
25
+ if isinstance(ref, dict):
26
+ return ref["run_id"]
27
+ return ref
28
+
29
+
30
+ def best_run(model: str, experiment: str, metric: str, mode: str = "max",
31
+ *, settings=None, backend=None) -> Optional[Dict[str, Any]]:
32
+ """Best child run of `experiment` under `model` by `metric` (None if no runs)."""
33
+ if mode not in ("max", "min"):
34
+ raise ValueError("mode must be 'max' or 'min'")
35
+ _, b = _ctx(settings, backend)
36
+ runs = b.search_runs(model, {"model": model, "experiment": experiment, TAG_LEVEL: "run"},
37
+ metric, mode, 1)
38
+ return runs[0] if runs else None
39
+
40
+
41
+ def register(run: Optional[RunRef] = None, name: Optional[str] = None, *,
42
+ settings=None, backend=None) -> str:
43
+ """Create a new model version from the model logged in `run` (default: the current run)."""
44
+ _, b = _ctx(settings, backend)
45
+ if run is None:
46
+ from .simple import current_run
47
+ run = current_run()
48
+ info = b.get_run(_run_id(run))
49
+ uri = info["tags"].get(TAG_MODEL_URI)
50
+ if not uri:
51
+ raise ValidationError("This run has no logged model; call run.log_model(...) first")
52
+ return b.register(info["run_id"], name or info["tags"]["model"], uri)
53
+
54
+
55
+ def _metric(b: Backend, name: str, version: str, metric: str) -> Optional[float]:
56
+ return b.get_run(b.version_run_id(name, version))["metrics"].get(metric)
57
+
58
+
59
+ def _check_gate(s: Settings, b: Backend, name: str, version: str, to: str) -> None:
60
+ if s.promote_metric is None:
61
+ return
62
+ value = _metric(b, name, version, s.promote_metric)
63
+ if value is None:
64
+ raise PromotionError(f"Version {version} has no metric {s.promote_metric!r}")
65
+ if s.promote_min is not None and value < s.promote_min:
66
+ raise PromotionError(
67
+ f"{s.promote_metric}={value} is below the minimum {s.promote_min} for {to!r}")
68
+ if s.promote_must_beat_prod:
69
+ current = b.get_alias(name, to)
70
+ if current is not None and current != version:
71
+ cur_value = _metric(b, name, current, s.promote_metric)
72
+ if cur_value is not None and value <= cur_value:
73
+ raise PromotionError(
74
+ f"{s.promote_metric}={value} does not beat current {to!r} "
75
+ f"(v{current}: {cur_value})")
76
+
77
+
78
+ def promote(name: str, version: Union[str, int], to: str, *, settings=None, backend=None) -> None:
79
+ """Point alias `to` at `version`. Gated aliases must pass the configured checks."""
80
+ s, b = _ctx(settings, backend)
81
+ version = str(version)
82
+ b.version_run_id(name, version) # raises if it does not exist
83
+ if to in s.gated_aliases:
84
+ _check_gate(s, b, name, version, to)
85
+ previous = b.get_alias(name, to)
86
+ if previous is not None and previous != version:
87
+ b.set_alias(name, previous, f"previous-{to}")
88
+ b.set_alias(name, version, to)
89
+
90
+
91
+ def rollback(name: str, alias: str = "production", *, settings=None, backend=None) -> str:
92
+ """Move `alias` back to the version it pointed at before the last promote."""
93
+ _, b = _ctx(settings, backend)
94
+ previous = b.get_alias(name, f"previous-{alias}")
95
+ if previous is None:
96
+ raise PromotionError(f"No previous version recorded for alias {alias!r}")
97
+ current = b.get_alias(name, alias)
98
+ b.set_alias(name, previous, alias)
99
+ if current is not None:
100
+ b.set_alias(name, current, f"previous-{alias}")
101
+ return previous
102
+
103
+
104
+ def load(name: str, alias: str = "production", *, device: str = "cpu", pickle: bool = False,
105
+ settings=None, backend=None) -> Any:
106
+ """Load a model by registry alias (never by run id).
107
+
108
+ A PyTorch bundle comes back as the real torch.nn.Module (on `device`, in eval mode); any other
109
+ model comes back as an MLflow model with predict().
110
+ """
111
+ _, b = _ctx(settings, backend)
112
+ folder = b.download_bundle(name, alias)
113
+ if folder is None:
114
+ return b.load(name, alias)
115
+ try:
116
+ return bundle.load_dir(folder, device=device, pickle=pickle)
117
+ finally:
118
+ shutil.rmtree(folder, ignore_errors=True)