authtracker 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
authtracker/simple.py ADDED
@@ -0,0 +1,241 @@
1
+ """The simple API: init -> start -> log -> save -> finish. No `with`, no run objects.
2
+
3
+ at.init(model="fake-image", experiment="baseline", dataset_version="v1") # level 2
4
+ at.start("lr0.01", config={"lr": 0.01}) # level 3
5
+ at.log({"loss": 0.4})
6
+ at.save(model)
7
+ at.finish() # optional
8
+
9
+ Level 1 (the model) is the MLflow experiment. Runs close themselves when the script ends;
10
+ if it ends with an unhandled exception they are marked FAILED with the traceback saved.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import atexit
15
+ import json
16
+ import os
17
+ import sys
18
+ from typing import Any, Dict, Mapping, Optional, Union
19
+
20
+ from . import configfile
21
+ from . import dataset as dataset_module
22
+ from .errors import AuthTrackerError, ValidationError
23
+ from .run import Run
24
+ from .run import run as _open_experiment
25
+ from .settings import Settings
26
+ from .watch import watch as _watch
27
+
28
+
29
+ class _State:
30
+ def __init__(self, cm, parent: Run, resume: bool = False):
31
+ self.parent_cm, self.parent = cm, parent
32
+ self.resume = resume # the default for start(): continue earlier runs of the same name
33
+ self.child_cm = None
34
+ self.child: Optional[Run] = None
35
+ self.count = 0 # child runs started (for default names)
36
+ self.steps: Dict[str, int] = {} # run id -> last auto step
37
+
38
+
39
+ _state: Optional[_State] = None
40
+ _crash: Optional[tuple] = None # (type, value, traceback) of an unhandled exception
41
+ _hooked = False
42
+
43
+
44
+ def _install_hooks() -> None:
45
+ global _hooked
46
+ if _hooked:
47
+ return
48
+ _hooked = True
49
+ previous = sys.excepthook
50
+
51
+ def hook(exc_type, exc, tb):
52
+ global _crash
53
+ _crash = (exc_type, exc, tb)
54
+ previous(exc_type, exc, tb)
55
+
56
+ sys.excepthook = hook
57
+ atexit.register(finish)
58
+
59
+
60
+ def _need() -> _State:
61
+ if _state is None:
62
+ raise AuthTrackerError("No active run: call authtracker.init(model=..., experiment=...) first")
63
+ return _state
64
+
65
+
66
+ def current_run() -> Run:
67
+ """The run that log()/save() write to: the current child, else the experiment run."""
68
+ s = _need()
69
+ return s.child or s.parent
70
+
71
+
72
+ def _continue_steps(s: _State, run: Run) -> None:
73
+ """After a resume, the automatic step carries on from the last step already logged."""
74
+ if run.resume_n:
75
+ last = run._backend.last_step(run.id)
76
+ if last is not None:
77
+ s.steps[run.id] = last
78
+
79
+
80
+ def _close_child(s: _State, exc=(None, None, None)) -> None:
81
+ if s.child_cm is not None:
82
+ cm, s.child_cm, s.child = s.child_cm, None, None
83
+ cm.__exit__(*exc)
84
+
85
+
86
+ def init(model: Optional[str] = None, experiment: Optional[str] = None, *,
87
+ config: Union[None, str, "os.PathLike[str]", Mapping[str, Any]] = None,
88
+ dataset: Union[None, str, "os.PathLike[str]", Mapping[str, Any]] = None,
89
+ dataset_version: Optional[str] = None, owner: Optional[str] = None,
90
+ tags: Optional[Mapping[str, str]] = None, save_code: Optional[bool] = None,
91
+ image: Optional[str] = None, system_metrics: Optional[bool] = None,
92
+ resume: bool = False, settings: Optional[Settings] = None) -> Run:
93
+ """Open the experiment run for `model`. Closes any run that is still open.
94
+
95
+ config a training YAML file (its path), or a dict of parameters. A YAML file is saved with the
96
+ run, its settings are logged as parameters, and an optional `authtracker:` block in it
97
+ can set model, experiment, dataset, tags and owner. Arguments given here win over it.
98
+ dataset the data this run uses: a path, or {"name", "version", "path", "hash"}. A fingerprint
99
+ (hash) of it is stored with the run.
100
+ save_code store the uncommitted changes and your script with the run. Default True: pass
101
+ save_code=False to keep them out (for example when the code holds secrets).
102
+ image the container image this runs in, ideally pinned with its digest:
103
+ image="registry/app@sha256:...". Recorded with the run; a tag alone gets a warning.
104
+ system_metrics record CPU, memory and GPU use. Default True: pass system_metrics=False to switch it off
105
+ (for example on the ranks of a multi-GPU job that are not tracking).
106
+ resume continue the latest earlier experiment run of this model and experiment (yours), if there is one,
107
+ and start a new one if not. The runs you start() afterwards continue earlier runs of the same
108
+ name by default; pass resume=False to start() to make a new one.
109
+ """
110
+ global _state
111
+ finish()
112
+ params: Dict[str, Any] = {}
113
+ block: Dict[str, Any] = {}
114
+ config_path: Optional[str] = None
115
+ if isinstance(config, (str, os.PathLike)):
116
+ params, block, config_path = configfile.load(config)
117
+ elif config:
118
+ params = configfile.flat_params(config)
119
+
120
+ model = model or block.get("model")
121
+ experiment = experiment or block.get("experiment")
122
+ if not model or not experiment:
123
+ raise ValidationError(
124
+ "model and experiment are required: pass them to init(...), or set them in the "
125
+ "'authtracker:' block of the config file")
126
+
127
+ spec = dataset if dataset is not None else block.get("dataset")
128
+ record = dataset_module.describe(spec) if spec is not None else None
129
+ version = dataset_version or block.get("dataset_version") or (record or {}).get("version")
130
+ all_tags: Dict[str, str] = {**(block.get("tags") or {}), **(tags or {}),
131
+ **(dataset_module.as_tags(record) if record else {})}
132
+
133
+ cm = _open_experiment(model, experiment, owner=owner or block.get("owner"),
134
+ dataset_version=version, tags=all_tags, save_code=save_code,
135
+ image=image, system_metrics=system_metrics, resume=resume, params=params, settings=settings)
136
+ parent = cm.__enter__()
137
+ _state = _State(cm, parent, resume)
138
+ _install_hooks()
139
+ _continue_steps(_state, parent)
140
+ if config_path: # the YAML itself, under its own name
141
+ with open(config_path, encoding="utf-8") as f:
142
+ parent.log_text(f.read(), parent._named(os.path.basename(config_path)))
143
+ if record:
144
+ parent.log_text(json.dumps(record, indent=2), parent._named("dataset.json"))
145
+ return parent
146
+
147
+
148
+ def start(name: Optional[str] = None, config: Optional[Mapping[str, Any]] = None,
149
+ resume: Optional[bool] = None) -> Run:
150
+ """Start a run (one trial) inside the experiment. Closes the previous run.
151
+
152
+ resume=True continues the latest earlier run of this name under this experiment run, or starts a new one
153
+ if there is none. Left out, it follows the `resume` you gave to init().
154
+ """
155
+ s = _need()
156
+ _close_child(s)
157
+ s.count += 1
158
+ cm = s.parent.child(name or f"run-{s.count}", params=config, resume=s.resume if resume is None else resume)
159
+ s.child = cm.__enter__()
160
+ s.child_cm = cm
161
+ _continue_steps(s, s.child)
162
+ return s.child
163
+
164
+
165
+ def log(metrics: Mapping[str, float], step: Optional[int] = None) -> None:
166
+ """Log metrics to the current run. `step` counts up by itself unless you pass it.
167
+
168
+ Metric names are checked against AUTHTRACKER_METRIC_NAMES when that is set.
169
+ """
170
+ s = _need()
171
+ r = s.child or s.parent
172
+ r.check_metric_names(metrics)
173
+ if step is None:
174
+ step = s.steps.get(r.id, -1) + 1
175
+ s.steps[r.id] = step
176
+ r.log_metrics(metrics, step)
177
+
178
+
179
+ def restore(name: str, folder: Optional[str] = None) -> Optional[str]:
180
+ """Download the file `name` that this run saved earlier (a checkpoint, say) and return its path.
181
+
182
+ Returns None if the run has no such file, so one script serves both the first run and every restart:
183
+
184
+ path = at.restore("ckpt.pt")
185
+ start = torch.load(path)["epoch"] + 1 if path else 0
186
+ """
187
+ return current_run().restore(name, folder)
188
+
189
+
190
+ def watch(model: Any, every: int = 100) -> Any:
191
+ """Log the gradient size of every layer of a PyTorch `model`, every `every` training steps.
192
+
193
+ Call it once, after init() and before training. It logs `grad_norm/<layer>` and `grad_norm/total`
194
+ (a step is one training forward pass). Returns a function that stops watching.
195
+ """
196
+ return _watch(model, every, current_run)
197
+
198
+
199
+ def set_tags(tags: Mapping[str, Any]) -> None:
200
+ current_run().set_tags(tags)
201
+
202
+
203
+ def save(obj: Any, name: Optional[str] = None, **kwargs: Any) -> Optional[str]:
204
+ """Save anything to the current run.
205
+
206
+ * a file path -> artifact
207
+ * a folder path -> every file in it, as a bundle
208
+ * a list of file paths -> a bundle
209
+ * text (with name=...) -> a text artifact such as a training log
210
+ * anything else -> a model (extra keyword arguments go to log_model)
211
+ """
212
+ r = current_run()
213
+ is_path = lambda x: isinstance(x, (str, os.PathLike)) # noqa: E731
214
+ if is_path(obj) and os.path.exists(obj):
215
+ path = os.fspath(obj)
216
+ r.log_artifacts(path) if os.path.isdir(path) else r.log_artifact(path)
217
+ return None
218
+ if isinstance(obj, (list, tuple)) and obj and all(is_path(x) and os.path.exists(x) for x in obj):
219
+ r.log_artifacts([os.fspath(x) for x in obj])
220
+ return None
221
+ if isinstance(obj, str):
222
+ if name is None:
223
+ raise ValidationError(
224
+ f"{obj!r} is not an existing file or folder. To save text, pass a file name: "
225
+ "save(text, name='train.log')")
226
+ r.log_text(obj, name)
227
+ return None
228
+ return r.log_model(obj, **kwargs)
229
+
230
+
231
+ def finish() -> None:
232
+ """Close the open runs. Optional: this also happens when the script ends."""
233
+ global _state
234
+ s, _state = _state, None
235
+ if s is None:
236
+ return
237
+ exc = _crash or (None, None, None)
238
+ try:
239
+ _close_child(s, exc)
240
+ finally:
241
+ s.parent_cm.__exit__(*exc)
authtracker/testing.py ADDED
@@ -0,0 +1,82 @@
1
+ """Contract checks for backends. Run them against every backend you write:
2
+
3
+ from authtracker.testing import check_backend
4
+
5
+ def test_my_backend():
6
+ check_backend(MyBackend(...)) # required behaviour
7
+ check_backend(MyBackend(...), model=my_model) # also model + registry, if supported
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import os
12
+ import tempfile
13
+ import uuid
14
+ from typing import Any, Optional
15
+
16
+ from .backends.base import TAG_LEVEL, Backend
17
+
18
+
19
+ def _supports(backend: Backend, method: str) -> bool:
20
+ inner = getattr(backend, "inner", backend) # look through the resilient wrapper
21
+ return getattr(type(inner), method) is not getattr(Backend, method)
22
+
23
+
24
+ def check_backend(backend: Backend, *, model: Optional[Any] = None) -> None:
25
+ exp = f"contract-{uuid.uuid4().hex[:8]}"
26
+ base_tags = {"model": exp, "experiment": "e"}
27
+
28
+ backend.ping()
29
+ parent = backend.start_run(exp, "parent", {**base_tags, TAG_LEVEL: "experiment"}, None)
30
+ assert isinstance(parent, str) and parent, "start_run must return a non-empty string id"
31
+
32
+ with tempfile.TemporaryDirectory() as d:
33
+ path = os.path.join(d, "note.txt")
34
+ with open(path, "w") as f:
35
+ f.write("hi")
36
+
37
+ ids = []
38
+ for i, (name, f1) in enumerate([("a", 0.7), ("b", 0.9), ("c", 0.8)]):
39
+ rid = backend.start_run(exp, name, {**base_tags, TAG_LEVEL: "run"}, parent)
40
+ ids.append(rid)
41
+ backend.log_params(rid, {"lr": 0.1 * (i + 1)})
42
+ backend.log_metrics(rid, {"loss": 1.0}, 0)
43
+ backend.log_metrics(rid, {"loss": 0.5}, 1)
44
+ backend.log_metrics(rid, {"f1": f1}, None)
45
+ backend.set_tags(rid, {"extra": "x"})
46
+ backend.log_artifact(rid, path)
47
+ if i == 0 and model is not None and _supports(backend, "log_model"):
48
+ _check_models(backend, rid, model, exp)
49
+ backend.end_run(rid, "FINISHED")
50
+
51
+ info = backend.get_run(ids[0])
52
+ assert {"run_id", "name", "metrics", "params", "tags"} <= set(info), "get_run keys"
53
+ assert info["run_id"] == ids[0]
54
+ assert info["name"] == "a", "get_run must report the run name"
55
+ assert info["metrics"]["loss"] == 0.5, "metrics must report the latest value"
56
+ assert abs(float(info["params"]["lr"]) - 0.1) < 1e-9
57
+ assert info["tags"]["extra"] == "x" and info["tags"]["model"] == exp
58
+
59
+ backend.end_run(parent, "FINISHED")
60
+
61
+ flt = {"model": exp, TAG_LEVEL: "run"}
62
+ best = backend.search_runs(exp, flt, "f1", "max", 2)
63
+ assert [r["metrics"]["f1"] for r in best] == [0.9, 0.8], "search_runs max order/limit"
64
+ assert backend.search_runs(exp, flt, "f1", "min", 1)[0]["metrics"]["f1"] == 0.7
65
+ assert len(backend.search_runs(exp, flt, "f1", "max", 10)) == 3, "parent run must be excluded"
66
+ assert backend.search_runs(exp, flt, "no_such_metric", "max", 5) == []
67
+ assert backend.search_runs(exp + "-missing", {}, "f1", "max", 1) == [], "unknown experiment"
68
+
69
+
70
+ def _check_models(backend: Backend, run_id: str, model: Any, name: str) -> None:
71
+ uri = backend.log_model(run_id, model, None)
72
+ assert uri, "log_model must return a URI"
73
+ assert backend.get_run(run_id)["tags"].get("authtracker.model_uri") == uri
74
+ if not _supports(backend, "register"):
75
+ return
76
+ version = backend.register(run_id, name, uri)
77
+ assert isinstance(version, str)
78
+ assert backend.get_alias(name, "production") is None
79
+ backend.set_alias(name, version, "production")
80
+ assert backend.get_alias(name, "production") == version
81
+ assert backend.version_run_id(name, version) == run_id
82
+ assert backend.load(name, "production") is not None
authtracker/utils.py ADDED
@@ -0,0 +1,321 @@
1
+ """Reproducibility capture: where the code lives, the environment, dependencies."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import logging
6
+ import os
7
+ import platform
8
+ import re
9
+ import socket
10
+ import subprocess
11
+ import sys
12
+ from typing import Any, Dict, List, Optional
13
+
14
+ _log = logging.getLogger("authtracker")
15
+
16
+ CODE_MAX_BYTES = 1_000_000 # a patch or a script bigger than this is not stored
17
+ UNTRACKED_MAX_BYTES = 200_000 # nor is a new file bigger than this inside the patch
18
+ PATCH_NAME = "code_changes.patch"
19
+
20
+ def _git(*args: str, cwd: Optional[str] = None) -> Optional[str]:
21
+ try:
22
+ out = subprocess.run(["git", *args], capture_output=True, text=True, timeout=5, cwd=cwd)
23
+ return out.stdout.strip() if out.returncode == 0 else None
24
+ except Exception:
25
+ return None
26
+
27
+
28
+ def _git_out(args: List[str], cwd: Optional[str], ok=(0,)) -> Optional[str]:
29
+ """stdout of a git command, or None if it failed. `ok` lists the exit codes that count as success."""
30
+ try:
31
+ out = subprocess.run(["git", *args], capture_output=True, text=True, timeout=20, cwd=cwd)
32
+ except Exception:
33
+ return None
34
+ return out.stdout if out.returncode in ok else None
35
+
36
+
37
+ def clean_remote(url: str) -> str:
38
+ """A git remote URL that is safe to store: no user name, password or token in it.
39
+
40
+ `https://user:ghp_secret@github.com/org/repo.git` -> `https://github.com/org/repo.git`.
41
+ """
42
+ return re.sub(r"^([a-z][a-z0-9+.-]*://)[^/@\s]+@", r"\1", url.strip(), flags=re.I)
43
+
44
+
45
+ def web_url(remote: str) -> Optional[str]:
46
+ """The repository's web address (https://host/org/repo) for a git remote, if it can be worked out."""
47
+ remote = clean_remote(remote)
48
+ match = re.match(r"^(?:ssh://)?(?:[\w.-]+@)?([\w.-]+)[:/](.+?)(?:\.git)?/?$", remote) \
49
+ if not remote.lower().startswith(("http://", "https://")) else None
50
+ if match:
51
+ return f"https://{match.group(1)}/{match.group(2)}"
52
+ match = re.match(r"^https?://([^/]+)/(.+?)(?:\.git)?/?$", remote, flags=re.I)
53
+ return f"https://{match.group(1)}/{match.group(2)}" if match else None
54
+
55
+
56
+ def commit_url(remote: str, sha: str) -> Optional[str]:
57
+ """A link to the commit on GitHub, GitLab or Bitbucket (None for other hosts)."""
58
+ base = web_url(remote)
59
+ if not base:
60
+ return None
61
+ host = base.split("/")[2].lower()
62
+ if "github" in host or "gitlab" in host:
63
+ return f"{base}/commit/{sha}"
64
+ if "bitbucket" in host:
65
+ return f"{base}/commits/{sha}"
66
+ return None
67
+
68
+
69
+ def _candidate_dirs() -> List[str]:
70
+ dirs = [os.getcwd()]
71
+ main_file = getattr(sys.modules.get("__main__"), "__file__", None) # the script you ran, if any
72
+ if main_file:
73
+ dirs.append(os.path.dirname(os.path.abspath(main_file)))
74
+ return list(dict.fromkeys(dirs))
75
+
76
+
77
+ def git_info(cwd: Optional[str] = None) -> Optional[Dict[str, str]]:
78
+ """Where the code came from: commit, branch, whether it has uncommitted changes, the remote it
79
+ can be fetched from and a link to the commit. None if the code is not in a git repository.
80
+
81
+ Looks in the current folder, then in the folder of the script that is running.
82
+ """
83
+ for folder in ([cwd] if cwd else _candidate_dirs()):
84
+ sha = _git("rev-parse", "HEAD", cwd=folder)
85
+ if not sha:
86
+ continue
87
+ info = {"sha": sha, "dirty": str(bool(_git("status", "--porcelain", cwd=folder))).lower()}
88
+ branch = _git("rev-parse", "--abbrev-ref", "HEAD", cwd=folder)
89
+ if branch and branch != "HEAD": # "HEAD" means a detached checkout: no branch
90
+ info["branch"] = branch
91
+ remote = _git("config", "--get", "remote.origin.url", cwd=folder)
92
+ if not remote:
93
+ names = (_git("remote", cwd=folder) or "").split()
94
+ remote = _git("config", "--get", f"remote.{names[0]}.url", cwd=folder) if names else None
95
+ if remote:
96
+ info["remote"] = clean_remote(remote)
97
+ link = commit_url(remote, sha)
98
+ if link:
99
+ info["url"] = link
100
+ return info
101
+ return None
102
+
103
+
104
+ # Things that should never be copied into a run: the code of a run is readable by everyone who can open it.
105
+ _SECRET_PATTERNS = [
106
+ ("an AWS access key", re.compile(r"\b(AKIA|ASIA)[0-9A-Z]{16}\b")),
107
+ ("a GitHub token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{30,}\b")),
108
+ ("a Slack token", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}")),
109
+ ("an API key (sk-...)", re.compile(r"\bsk-[A-Za-z0-9_-]{20,}")),
110
+ ("a Google API key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")),
111
+ ("a private key", re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY")),
112
+ ("a password or secret assigned in the code",
113
+ re.compile(r"""(?i)\b(?:password|passwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token)\w*\s*[:=]\s*['"][^'"\s]{8,}['"]""")),
114
+ ]
115
+
116
+
117
+ def find_secret(text: str) -> Optional[str]:
118
+ """What kind of secret `text` seems to contain (a best-effort check, not a guarantee), or None."""
119
+ for label, pattern in _SECRET_PATTERNS:
120
+ if pattern.search(text):
121
+ return label
122
+ return None
123
+
124
+
125
+ def _is_text(data: bytes) -> bool:
126
+ return b"\0" not in data[:8000]
127
+
128
+
129
+ def code_changes(folder: Optional[str] = None) -> Dict[str, Any]:
130
+ """The changes since the last commit as a patch you can `git apply`, new files included.
131
+
132
+ Returns {'patch': text or None, 'status': 'none' | 'saved' | 'skipped: <reason>', 'untracked_skipped': n}.
133
+ """
134
+ root = (_git_out(["rev-parse", "--show-toplevel"], folder) or "").strip()
135
+ if not root:
136
+ return {"patch": None, "status": "not a git repository", "untracked_skipped": 0}
137
+ pieces: List[str] = []
138
+ tracked = _git_out(["diff", "HEAD", "--no-color", "--no-ext-diff"], root)
139
+ if tracked:
140
+ pieces.append(tracked)
141
+ skipped = 0
142
+ for rel in filter(None, (_git_out(["ls-files", "--others", "--exclude-standard", "-z"], root) or "").split("\0")):
143
+ full = os.path.join(root, rel)
144
+ try:
145
+ if os.path.getsize(full) > UNTRACKED_MAX_BYTES or not _is_text(open(full, "rb").read(8000)):
146
+ skipped += 1
147
+ continue
148
+ except OSError:
149
+ continue
150
+ added = _git_out(["diff", "--no-index", "--no-color", "--no-ext-diff", "--", "/dev/null", rel], root, ok=(0, 1))
151
+ if added:
152
+ pieces.append(added)
153
+ patch = "".join(p if p.endswith("\n") else p + "\n" for p in pieces)
154
+ if not patch.strip():
155
+ return {"patch": None, "status": "none", "untracked_skipped": skipped}
156
+ if len(patch.encode("utf-8", "replace")) > CODE_MAX_BYTES:
157
+ return {"patch": None, "status": f"skipped: larger than {CODE_MAX_BYTES // 1000} KB", "untracked_skipped": skipped}
158
+ secret = find_secret(patch)
159
+ if secret:
160
+ return {"patch": None, "status": f"skipped: looks like it contains {secret}", "untracked_skipped": skipped}
161
+ return {"patch": patch, "status": "saved", "untracked_skipped": skipped}
162
+
163
+
164
+ def _scripts_dirs() -> set:
165
+ """Folders where installed console scripts live (`pytest`, `mlflow`, `uvicorn`, ...): never your own script."""
166
+ import sysconfig
167
+ found = {sysconfig.get_path("scripts"), os.path.join(sys.prefix, "bin"), os.path.join(sys.prefix, "Scripts")}
168
+ return {os.path.realpath(d) for d in found if d}
169
+
170
+
171
+ def script_snapshot() -> Dict[str, Any]:
172
+ """The script that is running (not a notebook, not a library): its text and its path in the repo."""
173
+ path = getattr(sys.modules.get("__main__"), "__file__", None)
174
+ if not path or not os.path.isfile(path):
175
+ return {"text": None, "status": "not a script (a notebook or the interactive prompt)"}
176
+ path = os.path.abspath(path)
177
+ stdlib = __import__("sysconfig").get_paths().get("stdlib", "\0")
178
+ if ("site-packages" in path or "dist-packages" in path or path.startswith(stdlib)
179
+ or os.path.realpath(os.path.dirname(path)) in _scripts_dirs()):
180
+ return {"text": None, "status": "not your script (a library's entry point)"}
181
+ if os.path.getsize(path) > CODE_MAX_BYTES:
182
+ return {"text": None, "status": f"skipped: larger than {CODE_MAX_BYTES // 1000} KB"}
183
+ try:
184
+ text = open(path, encoding="utf-8").read()
185
+ except (OSError, UnicodeDecodeError):
186
+ return {"text": None, "status": "skipped: not readable as text"}
187
+ secret = find_secret(text)
188
+ if secret:
189
+ return {"text": None, "status": f"skipped: looks like it contains {secret}"}
190
+ folder = os.path.dirname(path)
191
+ root = (_git_out(["rev-parse", "--show-toplevel"], folder) or "").strip()
192
+ relative = os.path.relpath(path, root) if root else None
193
+ return {"text": text, "status": "saved", "name": os.path.basename(path), "repo_path": relative}
194
+
195
+
196
+ # ---------------------------------------------------------------------------------------------- containers
197
+ _exists = os.path.exists # (a name tests can replace)
198
+ _DIGEST = re.compile(r"@(sha256:[0-9a-f]{64})$")
199
+ _warned_image: set = set()
200
+
201
+
202
+ def container_runtime() -> Optional[str]:
203
+ """"kubernetes", "docker", "podman" or "container" when this process runs inside a container, else None."""
204
+ if os.environ.get("KUBERNETES_SERVICE_HOST"):
205
+ return "kubernetes"
206
+ if _exists("/.dockerenv"):
207
+ return "docker"
208
+ if _exists("/run/.containerenv"):
209
+ return "podman"
210
+ try:
211
+ with open("/proc/1/cgroup", encoding="utf-8") as f:
212
+ if re.search(r"docker|kubepods|containerd|libpod|lxc", f.read()):
213
+ return "container"
214
+ except OSError:
215
+ pass
216
+ return None
217
+
218
+
219
+ def _ecs_image() -> Optional[Dict[str, str]]:
220
+ """On AWS ECS / Fargate the platform tells the container its own image and its digest."""
221
+ uri = os.environ.get("ECS_CONTAINER_METADATA_URI_V4")
222
+ if not uri:
223
+ return None
224
+ try:
225
+ import urllib.request
226
+ with urllib.request.urlopen(uri, timeout=2) as response:
227
+ data = json.load(response)
228
+ except Exception:
229
+ return None
230
+ found = {}
231
+ if data.get("Image"):
232
+ found["image"] = data["Image"]
233
+ if str(data.get("ImageID", "")).startswith("sha256:"):
234
+ found["digest"] = data["ImageID"]
235
+ return found or None
236
+
237
+
238
+ def _repository(reference: str) -> str:
239
+ """`registry.io/team/app:v3` -> `registry.io/team/app` (the part without a tag or digest)."""
240
+ name = reference.split("@")[0]
241
+ last = name.rsplit("/", 1)[-1]
242
+ return name[: -(len(last.split(":", 1)[1]) + 1)] if ":" in last else name
243
+
244
+
245
+ def _docker_digest(reference: str) -> Optional[str]:
246
+ """The digest a local image tag points to right now, if the docker command can tell (usually on the host)."""
247
+ try:
248
+ result = subprocess.run(["docker", "image", "inspect", "--format", "{{index .RepoDigests 0}}", reference],
249
+ capture_output=True, text=True, timeout=5)
250
+ except Exception:
251
+ return None
252
+ match = _DIGEST.search(result.stdout.strip()) if result.returncode == 0 else None
253
+ return match.group(1) if match else None
254
+
255
+
256
+ def container_info(image: Optional[str] = None) -> Optional[Dict[str, str]]:
257
+ """Which container image this ran in, pinned to a digest when that can be found out.
258
+
259
+ `image` is what you tell authtracker (`init(image=...)`). Otherwise it looks at the platform (AWS ECS)
260
+ and at CI variables (GitLab's CI_JOB_IMAGE). A container cannot normally see its own image name, so a
261
+ build or deploy step usually has to pass it in. Returns None outside a container when no image is given.
262
+ """
263
+ runtime = container_runtime()
264
+ found: Dict[str, str] = {}
265
+ if image:
266
+ found["image"] = image
267
+ else:
268
+ found = _ecs_image() or ({"image": os.environ["CI_JOB_IMAGE"]} if os.environ.get("CI_JOB_IMAGE") else {})
269
+ if not runtime and not found:
270
+ return None
271
+ info: Dict[str, str] = {"runtime": runtime or "not detected"}
272
+ reference = found.get("image")
273
+ digest = found.get("digest")
274
+ if reference:
275
+ match = _DIGEST.search(reference)
276
+ digest = digest or (match.group(1) if match else None) or _docker_digest(reference)
277
+ info["image"] = reference
278
+ if digest:
279
+ info["digest"] = digest
280
+ info["pinned"] = "true" if digest else "false"
281
+ if reference and digest and not _DIGEST.search(reference):
282
+ info["image_pinned"] = f"{_repository(reference)}@{digest}"
283
+ key = (reference, info["pinned"])
284
+ if info["pinned"] == "false" and key not in _warned_image:
285
+ _warned_image.add(key)
286
+ if reference:
287
+ _log.warning("The container image %r is not pinned to a digest, and a tag can point to a different "
288
+ "image later. Pass the digest to at.init(image='name@sha256:...') to pin it.", reference)
289
+ else:
290
+ _log.warning("This runs in a container, but its image is not known. Pass it to "
291
+ "at.init(image='registry/name@sha256:...') to record it.")
292
+ return info
293
+
294
+
295
+ def capture_env_tags() -> Dict[str, str]:
296
+ tags = {
297
+ "python_version": platform.python_version(),
298
+ "platform": platform.platform(),
299
+ "hostname": socket.gethostname(),
300
+ }
301
+ git = git_info()
302
+ if git:
303
+ tags["git_sha"] = git["sha"]
304
+ tags["git_dirty"] = git["dirty"]
305
+ for key in ("branch", "remote", "url"):
306
+ if key in git:
307
+ tags[f"git_{key}"] = git[key]
308
+ return tags
309
+
310
+
311
+ def requirements_text() -> str:
312
+ from importlib import metadata
313
+ lines = sorted(
314
+ f"{d.metadata['Name']}=={d.version}"
315
+ for d in metadata.distributions() if d.metadata["Name"]
316
+ )
317
+ return "\n".join(lines) + "\n"
318
+
319
+
320
+ def python_executable() -> str:
321
+ return sys.executable