authtracker 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- authtracker/__init__.py +16 -0
- authtracker/backends/__init__.py +95 -0
- authtracker/backends/base.py +136 -0
- authtracker/backends/local.py +200 -0
- authtracker/backends/mlflow_backend.py +310 -0
- authtracker/backends/resilient.py +329 -0
- authtracker/backends/wandb_backend.py +235 -0
- authtracker/bundle.py +887 -0
- authtracker/capture.py +175 -0
- authtracker/cli.py +51 -0
- authtracker/configfile.py +84 -0
- authtracker/dataset.py +98 -0
- authtracker/errors.py +18 -0
- authtracker/registry.py +118 -0
- authtracker/run.py +346 -0
- authtracker/settings.py +171 -0
- authtracker/simple.py +241 -0
- authtracker/testing.py +82 -0
- authtracker/utils.py +321 -0
- authtracker/watch.py +59 -0
- authtracker-0.0.1.dist-info/METADATA +215 -0
- authtracker-0.0.1.dist-info/RECORD +25 -0
- authtracker-0.0.1.dist-info/WHEEL +4 -0
- authtracker-0.0.1.dist-info/entry_points.txt +2 -0
- authtracker-0.0.1.dist-info/licenses/LICENSE +202 -0
authtracker/simple.py
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
"""The simple API: init -> start -> log -> save -> finish. No `with`, no run objects.
|
|
2
|
+
|
|
3
|
+
at.init(model="fake-image", experiment="baseline", dataset_version="v1") # level 2
|
|
4
|
+
at.start("lr0.01", config={"lr": 0.01}) # level 3
|
|
5
|
+
at.log({"loss": 0.4})
|
|
6
|
+
at.save(model)
|
|
7
|
+
at.finish() # optional
|
|
8
|
+
|
|
9
|
+
Level 1 (the model) is the MLflow experiment. Runs close themselves when the script ends;
|
|
10
|
+
if it ends with an unhandled exception they are marked FAILED with the traceback saved.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import atexit
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import sys
|
|
18
|
+
from typing import Any, Dict, Mapping, Optional, Union
|
|
19
|
+
|
|
20
|
+
from . import configfile
|
|
21
|
+
from . import dataset as dataset_module
|
|
22
|
+
from .errors import AuthTrackerError, ValidationError
|
|
23
|
+
from .run import Run
|
|
24
|
+
from .run import run as _open_experiment
|
|
25
|
+
from .settings import Settings
|
|
26
|
+
from .watch import watch as _watch
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class _State:
|
|
30
|
+
def __init__(self, cm, parent: Run, resume: bool = False):
|
|
31
|
+
self.parent_cm, self.parent = cm, parent
|
|
32
|
+
self.resume = resume # the default for start(): continue earlier runs of the same name
|
|
33
|
+
self.child_cm = None
|
|
34
|
+
self.child: Optional[Run] = None
|
|
35
|
+
self.count = 0 # child runs started (for default names)
|
|
36
|
+
self.steps: Dict[str, int] = {} # run id -> last auto step
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
_state: Optional[_State] = None
|
|
40
|
+
_crash: Optional[tuple] = None # (type, value, traceback) of an unhandled exception
|
|
41
|
+
_hooked = False
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _install_hooks() -> None:
|
|
45
|
+
global _hooked
|
|
46
|
+
if _hooked:
|
|
47
|
+
return
|
|
48
|
+
_hooked = True
|
|
49
|
+
previous = sys.excepthook
|
|
50
|
+
|
|
51
|
+
def hook(exc_type, exc, tb):
|
|
52
|
+
global _crash
|
|
53
|
+
_crash = (exc_type, exc, tb)
|
|
54
|
+
previous(exc_type, exc, tb)
|
|
55
|
+
|
|
56
|
+
sys.excepthook = hook
|
|
57
|
+
atexit.register(finish)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _need() -> _State:
|
|
61
|
+
if _state is None:
|
|
62
|
+
raise AuthTrackerError("No active run: call authtracker.init(model=..., experiment=...) first")
|
|
63
|
+
return _state
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def current_run() -> Run:
|
|
67
|
+
"""The run that log()/save() write to: the current child, else the experiment run."""
|
|
68
|
+
s = _need()
|
|
69
|
+
return s.child or s.parent
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _continue_steps(s: _State, run: Run) -> None:
|
|
73
|
+
"""After a resume, the automatic step carries on from the last step already logged."""
|
|
74
|
+
if run.resume_n:
|
|
75
|
+
last = run._backend.last_step(run.id)
|
|
76
|
+
if last is not None:
|
|
77
|
+
s.steps[run.id] = last
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _close_child(s: _State, exc=(None, None, None)) -> None:
|
|
81
|
+
if s.child_cm is not None:
|
|
82
|
+
cm, s.child_cm, s.child = s.child_cm, None, None
|
|
83
|
+
cm.__exit__(*exc)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def init(model: Optional[str] = None, experiment: Optional[str] = None, *,
|
|
87
|
+
config: Union[None, str, "os.PathLike[str]", Mapping[str, Any]] = None,
|
|
88
|
+
dataset: Union[None, str, "os.PathLike[str]", Mapping[str, Any]] = None,
|
|
89
|
+
dataset_version: Optional[str] = None, owner: Optional[str] = None,
|
|
90
|
+
tags: Optional[Mapping[str, str]] = None, save_code: Optional[bool] = None,
|
|
91
|
+
image: Optional[str] = None, system_metrics: Optional[bool] = None,
|
|
92
|
+
resume: bool = False, settings: Optional[Settings] = None) -> Run:
|
|
93
|
+
"""Open the experiment run for `model`. Closes any run that is still open.
|
|
94
|
+
|
|
95
|
+
config a training YAML file (its path), or a dict of parameters. A YAML file is saved with the
|
|
96
|
+
run, its settings are logged as parameters, and an optional `authtracker:` block in it
|
|
97
|
+
can set model, experiment, dataset, tags and owner. Arguments given here win over it.
|
|
98
|
+
dataset the data this run uses: a path, or {"name", "version", "path", "hash"}. A fingerprint
|
|
99
|
+
(hash) of it is stored with the run.
|
|
100
|
+
save_code store the uncommitted changes and your script with the run. Default True: pass
|
|
101
|
+
save_code=False to keep them out (for example when the code holds secrets).
|
|
102
|
+
image the container image this runs in, ideally pinned with its digest:
|
|
103
|
+
image="registry/app@sha256:...". Recorded with the run; a tag alone gets a warning.
|
|
104
|
+
system_metrics record CPU, memory and GPU use. Default True: pass system_metrics=False to switch it off
|
|
105
|
+
(for example on the ranks of a multi-GPU job that are not tracking).
|
|
106
|
+
resume continue the latest earlier experiment run of this model and experiment (yours), if there is one,
|
|
107
|
+
and start a new one if not. The runs you start() afterwards continue earlier runs of the same
|
|
108
|
+
name by default; pass resume=False to start() to make a new one.
|
|
109
|
+
"""
|
|
110
|
+
global _state
|
|
111
|
+
finish()
|
|
112
|
+
params: Dict[str, Any] = {}
|
|
113
|
+
block: Dict[str, Any] = {}
|
|
114
|
+
config_path: Optional[str] = None
|
|
115
|
+
if isinstance(config, (str, os.PathLike)):
|
|
116
|
+
params, block, config_path = configfile.load(config)
|
|
117
|
+
elif config:
|
|
118
|
+
params = configfile.flat_params(config)
|
|
119
|
+
|
|
120
|
+
model = model or block.get("model")
|
|
121
|
+
experiment = experiment or block.get("experiment")
|
|
122
|
+
if not model or not experiment:
|
|
123
|
+
raise ValidationError(
|
|
124
|
+
"model and experiment are required: pass them to init(...), or set them in the "
|
|
125
|
+
"'authtracker:' block of the config file")
|
|
126
|
+
|
|
127
|
+
spec = dataset if dataset is not None else block.get("dataset")
|
|
128
|
+
record = dataset_module.describe(spec) if spec is not None else None
|
|
129
|
+
version = dataset_version or block.get("dataset_version") or (record or {}).get("version")
|
|
130
|
+
all_tags: Dict[str, str] = {**(block.get("tags") or {}), **(tags or {}),
|
|
131
|
+
**(dataset_module.as_tags(record) if record else {})}
|
|
132
|
+
|
|
133
|
+
cm = _open_experiment(model, experiment, owner=owner or block.get("owner"),
|
|
134
|
+
dataset_version=version, tags=all_tags, save_code=save_code,
|
|
135
|
+
image=image, system_metrics=system_metrics, resume=resume, params=params, settings=settings)
|
|
136
|
+
parent = cm.__enter__()
|
|
137
|
+
_state = _State(cm, parent, resume)
|
|
138
|
+
_install_hooks()
|
|
139
|
+
_continue_steps(_state, parent)
|
|
140
|
+
if config_path: # the YAML itself, under its own name
|
|
141
|
+
with open(config_path, encoding="utf-8") as f:
|
|
142
|
+
parent.log_text(f.read(), parent._named(os.path.basename(config_path)))
|
|
143
|
+
if record:
|
|
144
|
+
parent.log_text(json.dumps(record, indent=2), parent._named("dataset.json"))
|
|
145
|
+
return parent
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def start(name: Optional[str] = None, config: Optional[Mapping[str, Any]] = None,
|
|
149
|
+
resume: Optional[bool] = None) -> Run:
|
|
150
|
+
"""Start a run (one trial) inside the experiment. Closes the previous run.
|
|
151
|
+
|
|
152
|
+
resume=True continues the latest earlier run of this name under this experiment run, or starts a new one
|
|
153
|
+
if there is none. Left out, it follows the `resume` you gave to init().
|
|
154
|
+
"""
|
|
155
|
+
s = _need()
|
|
156
|
+
_close_child(s)
|
|
157
|
+
s.count += 1
|
|
158
|
+
cm = s.parent.child(name or f"run-{s.count}", params=config, resume=s.resume if resume is None else resume)
|
|
159
|
+
s.child = cm.__enter__()
|
|
160
|
+
s.child_cm = cm
|
|
161
|
+
_continue_steps(s, s.child)
|
|
162
|
+
return s.child
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def log(metrics: Mapping[str, float], step: Optional[int] = None) -> None:
|
|
166
|
+
"""Log metrics to the current run. `step` counts up by itself unless you pass it.
|
|
167
|
+
|
|
168
|
+
Metric names are checked against AUTHTRACKER_METRIC_NAMES when that is set.
|
|
169
|
+
"""
|
|
170
|
+
s = _need()
|
|
171
|
+
r = s.child or s.parent
|
|
172
|
+
r.check_metric_names(metrics)
|
|
173
|
+
if step is None:
|
|
174
|
+
step = s.steps.get(r.id, -1) + 1
|
|
175
|
+
s.steps[r.id] = step
|
|
176
|
+
r.log_metrics(metrics, step)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def restore(name: str, folder: Optional[str] = None) -> Optional[str]:
|
|
180
|
+
"""Download the file `name` that this run saved earlier (a checkpoint, say) and return its path.
|
|
181
|
+
|
|
182
|
+
Returns None if the run has no such file, so one script serves both the first run and every restart:
|
|
183
|
+
|
|
184
|
+
path = at.restore("ckpt.pt")
|
|
185
|
+
start = torch.load(path)["epoch"] + 1 if path else 0
|
|
186
|
+
"""
|
|
187
|
+
return current_run().restore(name, folder)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def watch(model: Any, every: int = 100) -> Any:
|
|
191
|
+
"""Log the gradient size of every layer of a PyTorch `model`, every `every` training steps.
|
|
192
|
+
|
|
193
|
+
Call it once, after init() and before training. It logs `grad_norm/<layer>` and `grad_norm/total`
|
|
194
|
+
(a step is one training forward pass). Returns a function that stops watching.
|
|
195
|
+
"""
|
|
196
|
+
return _watch(model, every, current_run)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def set_tags(tags: Mapping[str, Any]) -> None:
|
|
200
|
+
current_run().set_tags(tags)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def save(obj: Any, name: Optional[str] = None, **kwargs: Any) -> Optional[str]:
|
|
204
|
+
"""Save anything to the current run.
|
|
205
|
+
|
|
206
|
+
* a file path -> artifact
|
|
207
|
+
* a folder path -> every file in it, as a bundle
|
|
208
|
+
* a list of file paths -> a bundle
|
|
209
|
+
* text (with name=...) -> a text artifact such as a training log
|
|
210
|
+
* anything else -> a model (extra keyword arguments go to log_model)
|
|
211
|
+
"""
|
|
212
|
+
r = current_run()
|
|
213
|
+
is_path = lambda x: isinstance(x, (str, os.PathLike)) # noqa: E731
|
|
214
|
+
if is_path(obj) and os.path.exists(obj):
|
|
215
|
+
path = os.fspath(obj)
|
|
216
|
+
r.log_artifacts(path) if os.path.isdir(path) else r.log_artifact(path)
|
|
217
|
+
return None
|
|
218
|
+
if isinstance(obj, (list, tuple)) and obj and all(is_path(x) and os.path.exists(x) for x in obj):
|
|
219
|
+
r.log_artifacts([os.fspath(x) for x in obj])
|
|
220
|
+
return None
|
|
221
|
+
if isinstance(obj, str):
|
|
222
|
+
if name is None:
|
|
223
|
+
raise ValidationError(
|
|
224
|
+
f"{obj!r} is not an existing file or folder. To save text, pass a file name: "
|
|
225
|
+
"save(text, name='train.log')")
|
|
226
|
+
r.log_text(obj, name)
|
|
227
|
+
return None
|
|
228
|
+
return r.log_model(obj, **kwargs)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def finish() -> None:
|
|
232
|
+
"""Close the open runs. Optional: this also happens when the script ends."""
|
|
233
|
+
global _state
|
|
234
|
+
s, _state = _state, None
|
|
235
|
+
if s is None:
|
|
236
|
+
return
|
|
237
|
+
exc = _crash or (None, None, None)
|
|
238
|
+
try:
|
|
239
|
+
_close_child(s, exc)
|
|
240
|
+
finally:
|
|
241
|
+
s.parent_cm.__exit__(*exc)
|
authtracker/testing.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Contract checks for backends. Run them against every backend you write:
|
|
2
|
+
|
|
3
|
+
from authtracker.testing import check_backend
|
|
4
|
+
|
|
5
|
+
def test_my_backend():
|
|
6
|
+
check_backend(MyBackend(...)) # required behaviour
|
|
7
|
+
check_backend(MyBackend(...), model=my_model) # also model + registry, if supported
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import tempfile
|
|
13
|
+
import uuid
|
|
14
|
+
from typing import Any, Optional
|
|
15
|
+
|
|
16
|
+
from .backends.base import TAG_LEVEL, Backend
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _supports(backend: Backend, method: str) -> bool:
|
|
20
|
+
inner = getattr(backend, "inner", backend) # look through the resilient wrapper
|
|
21
|
+
return getattr(type(inner), method) is not getattr(Backend, method)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def check_backend(backend: Backend, *, model: Optional[Any] = None) -> None:
|
|
25
|
+
exp = f"contract-{uuid.uuid4().hex[:8]}"
|
|
26
|
+
base_tags = {"model": exp, "experiment": "e"}
|
|
27
|
+
|
|
28
|
+
backend.ping()
|
|
29
|
+
parent = backend.start_run(exp, "parent", {**base_tags, TAG_LEVEL: "experiment"}, None)
|
|
30
|
+
assert isinstance(parent, str) and parent, "start_run must return a non-empty string id"
|
|
31
|
+
|
|
32
|
+
with tempfile.TemporaryDirectory() as d:
|
|
33
|
+
path = os.path.join(d, "note.txt")
|
|
34
|
+
with open(path, "w") as f:
|
|
35
|
+
f.write("hi")
|
|
36
|
+
|
|
37
|
+
ids = []
|
|
38
|
+
for i, (name, f1) in enumerate([("a", 0.7), ("b", 0.9), ("c", 0.8)]):
|
|
39
|
+
rid = backend.start_run(exp, name, {**base_tags, TAG_LEVEL: "run"}, parent)
|
|
40
|
+
ids.append(rid)
|
|
41
|
+
backend.log_params(rid, {"lr": 0.1 * (i + 1)})
|
|
42
|
+
backend.log_metrics(rid, {"loss": 1.0}, 0)
|
|
43
|
+
backend.log_metrics(rid, {"loss": 0.5}, 1)
|
|
44
|
+
backend.log_metrics(rid, {"f1": f1}, None)
|
|
45
|
+
backend.set_tags(rid, {"extra": "x"})
|
|
46
|
+
backend.log_artifact(rid, path)
|
|
47
|
+
if i == 0 and model is not None and _supports(backend, "log_model"):
|
|
48
|
+
_check_models(backend, rid, model, exp)
|
|
49
|
+
backend.end_run(rid, "FINISHED")
|
|
50
|
+
|
|
51
|
+
info = backend.get_run(ids[0])
|
|
52
|
+
assert {"run_id", "name", "metrics", "params", "tags"} <= set(info), "get_run keys"
|
|
53
|
+
assert info["run_id"] == ids[0]
|
|
54
|
+
assert info["name"] == "a", "get_run must report the run name"
|
|
55
|
+
assert info["metrics"]["loss"] == 0.5, "metrics must report the latest value"
|
|
56
|
+
assert abs(float(info["params"]["lr"]) - 0.1) < 1e-9
|
|
57
|
+
assert info["tags"]["extra"] == "x" and info["tags"]["model"] == exp
|
|
58
|
+
|
|
59
|
+
backend.end_run(parent, "FINISHED")
|
|
60
|
+
|
|
61
|
+
flt = {"model": exp, TAG_LEVEL: "run"}
|
|
62
|
+
best = backend.search_runs(exp, flt, "f1", "max", 2)
|
|
63
|
+
assert [r["metrics"]["f1"] for r in best] == [0.9, 0.8], "search_runs max order/limit"
|
|
64
|
+
assert backend.search_runs(exp, flt, "f1", "min", 1)[0]["metrics"]["f1"] == 0.7
|
|
65
|
+
assert len(backend.search_runs(exp, flt, "f1", "max", 10)) == 3, "parent run must be excluded"
|
|
66
|
+
assert backend.search_runs(exp, flt, "no_such_metric", "max", 5) == []
|
|
67
|
+
assert backend.search_runs(exp + "-missing", {}, "f1", "max", 1) == [], "unknown experiment"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _check_models(backend: Backend, run_id: str, model: Any, name: str) -> None:
|
|
71
|
+
uri = backend.log_model(run_id, model, None)
|
|
72
|
+
assert uri, "log_model must return a URI"
|
|
73
|
+
assert backend.get_run(run_id)["tags"].get("authtracker.model_uri") == uri
|
|
74
|
+
if not _supports(backend, "register"):
|
|
75
|
+
return
|
|
76
|
+
version = backend.register(run_id, name, uri)
|
|
77
|
+
assert isinstance(version, str)
|
|
78
|
+
assert backend.get_alias(name, "production") is None
|
|
79
|
+
backend.set_alias(name, version, "production")
|
|
80
|
+
assert backend.get_alias(name, "production") == version
|
|
81
|
+
assert backend.version_run_id(name, version) == run_id
|
|
82
|
+
assert backend.load(name, "production") is not None
|
authtracker/utils.py
ADDED
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
"""Reproducibility capture: where the code lives, the environment, dependencies."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import platform
|
|
8
|
+
import re
|
|
9
|
+
import socket
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
from typing import Any, Dict, List, Optional
|
|
13
|
+
|
|
14
|
+
_log = logging.getLogger("authtracker")
|
|
15
|
+
|
|
16
|
+
CODE_MAX_BYTES = 1_000_000 # a patch or a script bigger than this is not stored
|
|
17
|
+
UNTRACKED_MAX_BYTES = 200_000 # nor is a new file bigger than this inside the patch
|
|
18
|
+
PATCH_NAME = "code_changes.patch"
|
|
19
|
+
|
|
20
|
+
def _git(*args: str, cwd: Optional[str] = None) -> Optional[str]:
|
|
21
|
+
try:
|
|
22
|
+
out = subprocess.run(["git", *args], capture_output=True, text=True, timeout=5, cwd=cwd)
|
|
23
|
+
return out.stdout.strip() if out.returncode == 0 else None
|
|
24
|
+
except Exception:
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _git_out(args: List[str], cwd: Optional[str], ok=(0,)) -> Optional[str]:
|
|
29
|
+
"""stdout of a git command, or None if it failed. `ok` lists the exit codes that count as success."""
|
|
30
|
+
try:
|
|
31
|
+
out = subprocess.run(["git", *args], capture_output=True, text=True, timeout=20, cwd=cwd)
|
|
32
|
+
except Exception:
|
|
33
|
+
return None
|
|
34
|
+
return out.stdout if out.returncode in ok else None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def clean_remote(url: str) -> str:
|
|
38
|
+
"""A git remote URL that is safe to store: no user name, password or token in it.
|
|
39
|
+
|
|
40
|
+
`https://user:ghp_secret@github.com/org/repo.git` -> `https://github.com/org/repo.git`.
|
|
41
|
+
"""
|
|
42
|
+
return re.sub(r"^([a-z][a-z0-9+.-]*://)[^/@\s]+@", r"\1", url.strip(), flags=re.I)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def web_url(remote: str) -> Optional[str]:
|
|
46
|
+
"""The repository's web address (https://host/org/repo) for a git remote, if it can be worked out."""
|
|
47
|
+
remote = clean_remote(remote)
|
|
48
|
+
match = re.match(r"^(?:ssh://)?(?:[\w.-]+@)?([\w.-]+)[:/](.+?)(?:\.git)?/?$", remote) \
|
|
49
|
+
if not remote.lower().startswith(("http://", "https://")) else None
|
|
50
|
+
if match:
|
|
51
|
+
return f"https://{match.group(1)}/{match.group(2)}"
|
|
52
|
+
match = re.match(r"^https?://([^/]+)/(.+?)(?:\.git)?/?$", remote, flags=re.I)
|
|
53
|
+
return f"https://{match.group(1)}/{match.group(2)}" if match else None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def commit_url(remote: str, sha: str) -> Optional[str]:
|
|
57
|
+
"""A link to the commit on GitHub, GitLab or Bitbucket (None for other hosts)."""
|
|
58
|
+
base = web_url(remote)
|
|
59
|
+
if not base:
|
|
60
|
+
return None
|
|
61
|
+
host = base.split("/")[2].lower()
|
|
62
|
+
if "github" in host or "gitlab" in host:
|
|
63
|
+
return f"{base}/commit/{sha}"
|
|
64
|
+
if "bitbucket" in host:
|
|
65
|
+
return f"{base}/commits/{sha}"
|
|
66
|
+
return None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _candidate_dirs() -> List[str]:
|
|
70
|
+
dirs = [os.getcwd()]
|
|
71
|
+
main_file = getattr(sys.modules.get("__main__"), "__file__", None) # the script you ran, if any
|
|
72
|
+
if main_file:
|
|
73
|
+
dirs.append(os.path.dirname(os.path.abspath(main_file)))
|
|
74
|
+
return list(dict.fromkeys(dirs))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def git_info(cwd: Optional[str] = None) -> Optional[Dict[str, str]]:
|
|
78
|
+
"""Where the code came from: commit, branch, whether it has uncommitted changes, the remote it
|
|
79
|
+
can be fetched from and a link to the commit. None if the code is not in a git repository.
|
|
80
|
+
|
|
81
|
+
Looks in the current folder, then in the folder of the script that is running.
|
|
82
|
+
"""
|
|
83
|
+
for folder in ([cwd] if cwd else _candidate_dirs()):
|
|
84
|
+
sha = _git("rev-parse", "HEAD", cwd=folder)
|
|
85
|
+
if not sha:
|
|
86
|
+
continue
|
|
87
|
+
info = {"sha": sha, "dirty": str(bool(_git("status", "--porcelain", cwd=folder))).lower()}
|
|
88
|
+
branch = _git("rev-parse", "--abbrev-ref", "HEAD", cwd=folder)
|
|
89
|
+
if branch and branch != "HEAD": # "HEAD" means a detached checkout: no branch
|
|
90
|
+
info["branch"] = branch
|
|
91
|
+
remote = _git("config", "--get", "remote.origin.url", cwd=folder)
|
|
92
|
+
if not remote:
|
|
93
|
+
names = (_git("remote", cwd=folder) or "").split()
|
|
94
|
+
remote = _git("config", "--get", f"remote.{names[0]}.url", cwd=folder) if names else None
|
|
95
|
+
if remote:
|
|
96
|
+
info["remote"] = clean_remote(remote)
|
|
97
|
+
link = commit_url(remote, sha)
|
|
98
|
+
if link:
|
|
99
|
+
info["url"] = link
|
|
100
|
+
return info
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# Things that should never be copied into a run: the code of a run is readable by everyone who can open it.
|
|
105
|
+
_SECRET_PATTERNS = [
|
|
106
|
+
("an AWS access key", re.compile(r"\b(AKIA|ASIA)[0-9A-Z]{16}\b")),
|
|
107
|
+
("a GitHub token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{30,}\b")),
|
|
108
|
+
("a Slack token", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}")),
|
|
109
|
+
("an API key (sk-...)", re.compile(r"\bsk-[A-Za-z0-9_-]{20,}")),
|
|
110
|
+
("a Google API key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")),
|
|
111
|
+
("a private key", re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY")),
|
|
112
|
+
("a password or secret assigned in the code",
|
|
113
|
+
re.compile(r"""(?i)\b(?:password|passwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token)\w*\s*[:=]\s*['"][^'"\s]{8,}['"]""")),
|
|
114
|
+
]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def find_secret(text: str) -> Optional[str]:
|
|
118
|
+
"""What kind of secret `text` seems to contain (a best-effort check, not a guarantee), or None."""
|
|
119
|
+
for label, pattern in _SECRET_PATTERNS:
|
|
120
|
+
if pattern.search(text):
|
|
121
|
+
return label
|
|
122
|
+
return None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _is_text(data: bytes) -> bool:
|
|
126
|
+
return b"\0" not in data[:8000]
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def code_changes(folder: Optional[str] = None) -> Dict[str, Any]:
|
|
130
|
+
"""The changes since the last commit as a patch you can `git apply`, new files included.
|
|
131
|
+
|
|
132
|
+
Returns {'patch': text or None, 'status': 'none' | 'saved' | 'skipped: <reason>', 'untracked_skipped': n}.
|
|
133
|
+
"""
|
|
134
|
+
root = (_git_out(["rev-parse", "--show-toplevel"], folder) or "").strip()
|
|
135
|
+
if not root:
|
|
136
|
+
return {"patch": None, "status": "not a git repository", "untracked_skipped": 0}
|
|
137
|
+
pieces: List[str] = []
|
|
138
|
+
tracked = _git_out(["diff", "HEAD", "--no-color", "--no-ext-diff"], root)
|
|
139
|
+
if tracked:
|
|
140
|
+
pieces.append(tracked)
|
|
141
|
+
skipped = 0
|
|
142
|
+
for rel in filter(None, (_git_out(["ls-files", "--others", "--exclude-standard", "-z"], root) or "").split("\0")):
|
|
143
|
+
full = os.path.join(root, rel)
|
|
144
|
+
try:
|
|
145
|
+
if os.path.getsize(full) > UNTRACKED_MAX_BYTES or not _is_text(open(full, "rb").read(8000)):
|
|
146
|
+
skipped += 1
|
|
147
|
+
continue
|
|
148
|
+
except OSError:
|
|
149
|
+
continue
|
|
150
|
+
added = _git_out(["diff", "--no-index", "--no-color", "--no-ext-diff", "--", "/dev/null", rel], root, ok=(0, 1))
|
|
151
|
+
if added:
|
|
152
|
+
pieces.append(added)
|
|
153
|
+
patch = "".join(p if p.endswith("\n") else p + "\n" for p in pieces)
|
|
154
|
+
if not patch.strip():
|
|
155
|
+
return {"patch": None, "status": "none", "untracked_skipped": skipped}
|
|
156
|
+
if len(patch.encode("utf-8", "replace")) > CODE_MAX_BYTES:
|
|
157
|
+
return {"patch": None, "status": f"skipped: larger than {CODE_MAX_BYTES // 1000} KB", "untracked_skipped": skipped}
|
|
158
|
+
secret = find_secret(patch)
|
|
159
|
+
if secret:
|
|
160
|
+
return {"patch": None, "status": f"skipped: looks like it contains {secret}", "untracked_skipped": skipped}
|
|
161
|
+
return {"patch": patch, "status": "saved", "untracked_skipped": skipped}
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _scripts_dirs() -> set:
|
|
165
|
+
"""Folders where installed console scripts live (`pytest`, `mlflow`, `uvicorn`, ...): never your own script."""
|
|
166
|
+
import sysconfig
|
|
167
|
+
found = {sysconfig.get_path("scripts"), os.path.join(sys.prefix, "bin"), os.path.join(sys.prefix, "Scripts")}
|
|
168
|
+
return {os.path.realpath(d) for d in found if d}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def script_snapshot() -> Dict[str, Any]:
|
|
172
|
+
"""The script that is running (not a notebook, not a library): its text and its path in the repo."""
|
|
173
|
+
path = getattr(sys.modules.get("__main__"), "__file__", None)
|
|
174
|
+
if not path or not os.path.isfile(path):
|
|
175
|
+
return {"text": None, "status": "not a script (a notebook or the interactive prompt)"}
|
|
176
|
+
path = os.path.abspath(path)
|
|
177
|
+
stdlib = __import__("sysconfig").get_paths().get("stdlib", "\0")
|
|
178
|
+
if ("site-packages" in path or "dist-packages" in path or path.startswith(stdlib)
|
|
179
|
+
or os.path.realpath(os.path.dirname(path)) in _scripts_dirs()):
|
|
180
|
+
return {"text": None, "status": "not your script (a library's entry point)"}
|
|
181
|
+
if os.path.getsize(path) > CODE_MAX_BYTES:
|
|
182
|
+
return {"text": None, "status": f"skipped: larger than {CODE_MAX_BYTES // 1000} KB"}
|
|
183
|
+
try:
|
|
184
|
+
text = open(path, encoding="utf-8").read()
|
|
185
|
+
except (OSError, UnicodeDecodeError):
|
|
186
|
+
return {"text": None, "status": "skipped: not readable as text"}
|
|
187
|
+
secret = find_secret(text)
|
|
188
|
+
if secret:
|
|
189
|
+
return {"text": None, "status": f"skipped: looks like it contains {secret}"}
|
|
190
|
+
folder = os.path.dirname(path)
|
|
191
|
+
root = (_git_out(["rev-parse", "--show-toplevel"], folder) or "").strip()
|
|
192
|
+
relative = os.path.relpath(path, root) if root else None
|
|
193
|
+
return {"text": text, "status": "saved", "name": os.path.basename(path), "repo_path": relative}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
# ---------------------------------------------------------------------------------------------- containers
|
|
197
|
+
_exists = os.path.exists # (a name tests can replace)
|
|
198
|
+
_DIGEST = re.compile(r"@(sha256:[0-9a-f]{64})$")
|
|
199
|
+
_warned_image: set = set()
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def container_runtime() -> Optional[str]:
|
|
203
|
+
""""kubernetes", "docker", "podman" or "container" when this process runs inside a container, else None."""
|
|
204
|
+
if os.environ.get("KUBERNETES_SERVICE_HOST"):
|
|
205
|
+
return "kubernetes"
|
|
206
|
+
if _exists("/.dockerenv"):
|
|
207
|
+
return "docker"
|
|
208
|
+
if _exists("/run/.containerenv"):
|
|
209
|
+
return "podman"
|
|
210
|
+
try:
|
|
211
|
+
with open("/proc/1/cgroup", encoding="utf-8") as f:
|
|
212
|
+
if re.search(r"docker|kubepods|containerd|libpod|lxc", f.read()):
|
|
213
|
+
return "container"
|
|
214
|
+
except OSError:
|
|
215
|
+
pass
|
|
216
|
+
return None
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _ecs_image() -> Optional[Dict[str, str]]:
|
|
220
|
+
"""On AWS ECS / Fargate the platform tells the container its own image and its digest."""
|
|
221
|
+
uri = os.environ.get("ECS_CONTAINER_METADATA_URI_V4")
|
|
222
|
+
if not uri:
|
|
223
|
+
return None
|
|
224
|
+
try:
|
|
225
|
+
import urllib.request
|
|
226
|
+
with urllib.request.urlopen(uri, timeout=2) as response:
|
|
227
|
+
data = json.load(response)
|
|
228
|
+
except Exception:
|
|
229
|
+
return None
|
|
230
|
+
found = {}
|
|
231
|
+
if data.get("Image"):
|
|
232
|
+
found["image"] = data["Image"]
|
|
233
|
+
if str(data.get("ImageID", "")).startswith("sha256:"):
|
|
234
|
+
found["digest"] = data["ImageID"]
|
|
235
|
+
return found or None
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _repository(reference: str) -> str:
|
|
239
|
+
"""`registry.io/team/app:v3` -> `registry.io/team/app` (the part without a tag or digest)."""
|
|
240
|
+
name = reference.split("@")[0]
|
|
241
|
+
last = name.rsplit("/", 1)[-1]
|
|
242
|
+
return name[: -(len(last.split(":", 1)[1]) + 1)] if ":" in last else name
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _docker_digest(reference: str) -> Optional[str]:
|
|
246
|
+
"""The digest a local image tag points to right now, if the docker command can tell (usually on the host)."""
|
|
247
|
+
try:
|
|
248
|
+
result = subprocess.run(["docker", "image", "inspect", "--format", "{{index .RepoDigests 0}}", reference],
|
|
249
|
+
capture_output=True, text=True, timeout=5)
|
|
250
|
+
except Exception:
|
|
251
|
+
return None
|
|
252
|
+
match = _DIGEST.search(result.stdout.strip()) if result.returncode == 0 else None
|
|
253
|
+
return match.group(1) if match else None
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def container_info(image: Optional[str] = None) -> Optional[Dict[str, str]]:
|
|
257
|
+
"""Which container image this ran in, pinned to a digest when that can be found out.
|
|
258
|
+
|
|
259
|
+
`image` is what you tell authtracker (`init(image=...)`). Otherwise it looks at the platform (AWS ECS)
|
|
260
|
+
and at CI variables (GitLab's CI_JOB_IMAGE). A container cannot normally see its own image name, so a
|
|
261
|
+
build or deploy step usually has to pass it in. Returns None outside a container when no image is given.
|
|
262
|
+
"""
|
|
263
|
+
runtime = container_runtime()
|
|
264
|
+
found: Dict[str, str] = {}
|
|
265
|
+
if image:
|
|
266
|
+
found["image"] = image
|
|
267
|
+
else:
|
|
268
|
+
found = _ecs_image() or ({"image": os.environ["CI_JOB_IMAGE"]} if os.environ.get("CI_JOB_IMAGE") else {})
|
|
269
|
+
if not runtime and not found:
|
|
270
|
+
return None
|
|
271
|
+
info: Dict[str, str] = {"runtime": runtime or "not detected"}
|
|
272
|
+
reference = found.get("image")
|
|
273
|
+
digest = found.get("digest")
|
|
274
|
+
if reference:
|
|
275
|
+
match = _DIGEST.search(reference)
|
|
276
|
+
digest = digest or (match.group(1) if match else None) or _docker_digest(reference)
|
|
277
|
+
info["image"] = reference
|
|
278
|
+
if digest:
|
|
279
|
+
info["digest"] = digest
|
|
280
|
+
info["pinned"] = "true" if digest else "false"
|
|
281
|
+
if reference and digest and not _DIGEST.search(reference):
|
|
282
|
+
info["image_pinned"] = f"{_repository(reference)}@{digest}"
|
|
283
|
+
key = (reference, info["pinned"])
|
|
284
|
+
if info["pinned"] == "false" and key not in _warned_image:
|
|
285
|
+
_warned_image.add(key)
|
|
286
|
+
if reference:
|
|
287
|
+
_log.warning("The container image %r is not pinned to a digest, and a tag can point to a different "
|
|
288
|
+
"image later. Pass the digest to at.init(image='name@sha256:...') to pin it.", reference)
|
|
289
|
+
else:
|
|
290
|
+
_log.warning("This runs in a container, but its image is not known. Pass it to "
|
|
291
|
+
"at.init(image='registry/name@sha256:...') to record it.")
|
|
292
|
+
return info
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def capture_env_tags() -> Dict[str, str]:
|
|
296
|
+
tags = {
|
|
297
|
+
"python_version": platform.python_version(),
|
|
298
|
+
"platform": platform.platform(),
|
|
299
|
+
"hostname": socket.gethostname(),
|
|
300
|
+
}
|
|
301
|
+
git = git_info()
|
|
302
|
+
if git:
|
|
303
|
+
tags["git_sha"] = git["sha"]
|
|
304
|
+
tags["git_dirty"] = git["dirty"]
|
|
305
|
+
for key in ("branch", "remote", "url"):
|
|
306
|
+
if key in git:
|
|
307
|
+
tags[f"git_{key}"] = git[key]
|
|
308
|
+
return tags
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def requirements_text() -> str:
|
|
312
|
+
from importlib import metadata
|
|
313
|
+
lines = sorted(
|
|
314
|
+
f"{d.metadata['Name']}=={d.version}"
|
|
315
|
+
for d in metadata.distributions() if d.metadata["Name"]
|
|
316
|
+
)
|
|
317
|
+
return "\n".join(lines) + "\n"
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def python_executable() -> str:
|
|
321
|
+
return sys.executable
|