airon 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airon/__init__.py +30 -0
- airon/__main__.py +5 -0
- airon/_config.py +138 -0
- airon/_console.py +125 -0
- airon/_env.py +163 -0
- airon/_hooks.py +116 -0
- airon/_log.py +54 -0
- airon/_names.py +81 -0
- airon/_run.py +672 -0
- airon/_sender.py +454 -0
- airon/_spool.py +199 -0
- airon/_sync.py +110 -0
- airon/_system.py +302 -0
- airon/_transport.py +242 -0
- airon/_values.py +132 -0
- airon/_version.py +4 -0
- airon/cli.py +330 -0
- airon/py.typed +0 -0
- airon-0.1.0.dist-info/METADATA +212 -0
- airon-0.1.0.dist-info/RECORD +23 -0
- airon-0.1.0.dist-info/WHEEL +4 -0
- airon-0.1.0.dist-info/entry_points.txt +2 -0
- airon-0.1.0.dist-info/licenses/LICENSE +21 -0
airon/__init__.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Airon: track machine learning runs without getting in the way of training.
|
|
2
|
+
|
|
3
|
+
import airon
|
|
4
|
+
|
|
5
|
+
run = airon.init(project="my-project", total_steps=50_000)
|
|
6
|
+
run.log({"train/loss": 0.4}, step=100)
|
|
7
|
+
run.finish()
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import TYPE_CHECKING, Any
|
|
13
|
+
|
|
14
|
+
from . import _run
|
|
15
|
+
from ._run import NoopRun as NoopRun
|
|
16
|
+
from ._run import Run as Run
|
|
17
|
+
from ._run import finish as finish
|
|
18
|
+
from ._run import init as init
|
|
19
|
+
from ._run import log as log
|
|
20
|
+
from ._version import __version__ as __version__
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
run: Run | None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def __getattr__(name: str) -> Any:
|
|
27
|
+
# ``airon.run`` is the current run, like ``wandb.run``; it changes on init and finish.
|
|
28
|
+
if name == "run":
|
|
29
|
+
return _run.current()
|
|
30
|
+
raise AttributeError(f"module 'airon' has no attribute {name!r}")
|
airon/__main__.py
ADDED
airon/_config.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Where settings come from: arguments, then environment variables, then the credentials file."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import tempfile
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
from urllib.parse import urlsplit
|
|
11
|
+
|
|
12
|
+
MODES = ("online", "offline", "disabled")
|
|
13
|
+
RESUME_MODES = ("never", "allow", "must")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def env(name: str) -> str | None:
|
|
17
|
+
value = os.environ.get(name)
|
|
18
|
+
return value.strip() if value and value.strip() else None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def env_flag(name: str) -> bool:
|
|
22
|
+
return (env(name) or "").lower() in ("1", "true", "yes", "on")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _home() -> Path:
|
|
26
|
+
try:
|
|
27
|
+
return Path.home()
|
|
28
|
+
except Exception:
|
|
29
|
+
return Path(tempfile.gettempdir())
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def airon_dir() -> Path | None:
|
|
33
|
+
value = env("AIRON_DIR")
|
|
34
|
+
return Path(value).expanduser() if value else None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def spool_root() -> Path:
|
|
38
|
+
"""Directory holding one sub-directory per spooled run."""
|
|
39
|
+
base = airon_dir() or (_home() / ".cache" / "airon")
|
|
40
|
+
return base / "runs"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def credentials_path() -> Path:
|
|
44
|
+
base = airon_dir() or (_home() / ".config" / "airon")
|
|
45
|
+
return base / "credentials.json"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def load_credentials() -> dict[str, str]:
|
|
49
|
+
try:
|
|
50
|
+
data = json.loads(credentials_path().read_text(encoding="utf-8"))
|
|
51
|
+
except Exception:
|
|
52
|
+
return {}
|
|
53
|
+
if not isinstance(data, dict):
|
|
54
|
+
return {}
|
|
55
|
+
return {k: v for k, v in data.items() if isinstance(v, str)}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def save_credentials(url: str, api_key: str) -> Path:
|
|
59
|
+
"""Write the credentials file readable by its owner only."""
|
|
60
|
+
path = credentials_path()
|
|
61
|
+
created = not path.parent.exists()
|
|
62
|
+
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
63
|
+
if created:
|
|
64
|
+
os.chmod(path.parent, 0o700)
|
|
65
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
66
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
67
|
+
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
68
|
+
json.dump({"url": url, "api_key": api_key}, handle)
|
|
69
|
+
os.chmod(tmp, 0o600)
|
|
70
|
+
os.replace(tmp, path)
|
|
71
|
+
return path
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def delete_credentials() -> bool:
|
|
75
|
+
try:
|
|
76
|
+
credentials_path().unlink()
|
|
77
|
+
except FileNotFoundError:
|
|
78
|
+
return False
|
|
79
|
+
return True
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def normalize_url(raw: str | None) -> str | None:
|
|
83
|
+
"""Return ``scheme://host[:port][/prefix]`` without a trailing slash, or None if unusable."""
|
|
84
|
+
if not raw:
|
|
85
|
+
return None
|
|
86
|
+
text = raw.strip()
|
|
87
|
+
if "://" not in text:
|
|
88
|
+
text = "https://" + text
|
|
89
|
+
try:
|
|
90
|
+
parts = urlsplit(text)
|
|
91
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
92
|
+
return None
|
|
93
|
+
parts.port # noqa: B018 - raises ValueError for a malformed port
|
|
94
|
+
except ValueError:
|
|
95
|
+
return None
|
|
96
|
+
return f"{parts.scheme}://{parts.netloc}{parts.path}".rstrip("/")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class Connection:
|
|
100
|
+
"""The resolved server address and key, and where each came from."""
|
|
101
|
+
|
|
102
|
+
def __init__(
|
|
103
|
+
self,
|
|
104
|
+
url: str | None,
|
|
105
|
+
api_key: str | None,
|
|
106
|
+
url_source: str,
|
|
107
|
+
key_source: str,
|
|
108
|
+
raw_url: str | None,
|
|
109
|
+
):
|
|
110
|
+
self.url = url
|
|
111
|
+
self.raw_url = raw_url
|
|
112
|
+
self.api_key = api_key
|
|
113
|
+
self.url_source = url_source
|
|
114
|
+
self.key_source = key_source
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def complete(self) -> bool:
|
|
118
|
+
return bool(self.url and self.api_key)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def resolve_connection(url: str | None = None, api_key: str | None = None) -> Connection:
|
|
122
|
+
stored: dict[str, Any] | None = None
|
|
123
|
+
|
|
124
|
+
def pick(argument: str | None, variable: str, field: str) -> tuple[str | None, str]:
|
|
125
|
+
nonlocal stored
|
|
126
|
+
if argument:
|
|
127
|
+
return argument, "argument"
|
|
128
|
+
if (value := env(variable)) is not None:
|
|
129
|
+
return value, f"environment variable {variable}"
|
|
130
|
+
if stored is None:
|
|
131
|
+
stored = load_credentials()
|
|
132
|
+
if stored.get(field):
|
|
133
|
+
return stored[field], str(credentials_path())
|
|
134
|
+
return None, "not set"
|
|
135
|
+
|
|
136
|
+
raw_url, url_source = pick(url, "AIRON_URL", "url")
|
|
137
|
+
key, key_source = pick(api_key, "AIRON_API_KEY", "api_key")
|
|
138
|
+
return Connection(normalize_url(raw_url), key, url_source, key_source, raw_url)
|
airon/_console.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Capture the tail of stdout and stderr without changing what the program prints."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import sys
|
|
7
|
+
import threading
|
|
8
|
+
from collections import deque
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
LINE_LIMIT = 4000
|
|
12
|
+
_SPLIT = re.compile(r"(\r\n|\n|\r)")
|
|
13
|
+
_ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class _Tee:
|
|
17
|
+
"""Forwards everything to the real stream and shows the text to the capture."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, original: Any, name: str, capture: ConsoleCapture):
|
|
20
|
+
self._original = original
|
|
21
|
+
self._name = name
|
|
22
|
+
self._capture = capture
|
|
23
|
+
|
|
24
|
+
def write(self, text: str) -> int:
|
|
25
|
+
written = self._original.write(text)
|
|
26
|
+
try:
|
|
27
|
+
self._capture.feed(self._name, text)
|
|
28
|
+
except Exception:
|
|
29
|
+
pass # capturing must never turn into an error in the program's print()
|
|
30
|
+
return len(text) if written is None else int(written)
|
|
31
|
+
|
|
32
|
+
def writelines(self, lines: Any) -> None:
|
|
33
|
+
for line in lines:
|
|
34
|
+
self.write(line)
|
|
35
|
+
|
|
36
|
+
def __getattr__(self, name: str) -> Any:
|
|
37
|
+
return getattr(self._original, name)
|
|
38
|
+
|
|
39
|
+
def __iter__(self) -> Any:
|
|
40
|
+
return iter(self._original)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class _Assembler:
|
|
44
|
+
"""Builds lines from a stream of text, treating a lone carriage return as "start over"."""
|
|
45
|
+
|
|
46
|
+
def __init__(self) -> None:
|
|
47
|
+
self.parts: list[str] = []
|
|
48
|
+
self.overwrite = False
|
|
49
|
+
|
|
50
|
+
def feed(self, text: str) -> list[str]:
|
|
51
|
+
done: list[str] = []
|
|
52
|
+
for piece in _SPLIT.split(text):
|
|
53
|
+
if piece in ("\n", "\r\n"):
|
|
54
|
+
done.append("".join(self.parts))
|
|
55
|
+
self.parts = []
|
|
56
|
+
self.overwrite = False
|
|
57
|
+
elif piece == "\r":
|
|
58
|
+
self.overwrite = True
|
|
59
|
+
elif piece:
|
|
60
|
+
if self.overwrite:
|
|
61
|
+
self.parts = []
|
|
62
|
+
self.overwrite = False
|
|
63
|
+
self.parts.append(piece)
|
|
64
|
+
return done
|
|
65
|
+
|
|
66
|
+
def rest(self) -> str:
|
|
67
|
+
text = "".join(self.parts)
|
|
68
|
+
self.parts = []
|
|
69
|
+
return text
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class ConsoleCapture:
|
|
73
|
+
def __init__(self, max_lines: int = 200):
|
|
74
|
+
self._lock = threading.RLock()
|
|
75
|
+
self._lines: deque[tuple[str, str]] = deque(maxlen=max(1, max_lines))
|
|
76
|
+
self._assemblers = {"stdout": _Assembler(), "stderr": _Assembler()}
|
|
77
|
+
self._omitted = 0
|
|
78
|
+
self._tees: dict[str, _Tee] = {}
|
|
79
|
+
self._originals: dict[str, Any] = {}
|
|
80
|
+
self._active = False
|
|
81
|
+
|
|
82
|
+
def install(self) -> None:
|
|
83
|
+
for name in ("stdout", "stderr"):
|
|
84
|
+
original = getattr(sys, name)
|
|
85
|
+
self._originals[name] = original
|
|
86
|
+
tee = _Tee(original, name, self)
|
|
87
|
+
self._tees[name] = tee
|
|
88
|
+
setattr(sys, name, tee)
|
|
89
|
+
self._active = True
|
|
90
|
+
|
|
91
|
+
def uninstall(self) -> None:
|
|
92
|
+
for name, tee in self._tees.items():
|
|
93
|
+
if getattr(sys, name) is tee:
|
|
94
|
+
setattr(sys, name, self._originals[name])
|
|
95
|
+
with self._lock:
|
|
96
|
+
self._active = False
|
|
97
|
+
for name, assembler in self._assemblers.items():
|
|
98
|
+
self._add(name, assembler.rest())
|
|
99
|
+
|
|
100
|
+
def feed(self, stream: str, text: str) -> None:
|
|
101
|
+
if not self._active:
|
|
102
|
+
return
|
|
103
|
+
with self._lock:
|
|
104
|
+
for line in self._assemblers[stream].feed(text):
|
|
105
|
+
self._add(stream, line)
|
|
106
|
+
|
|
107
|
+
def _add(self, stream: str, line: str) -> None:
|
|
108
|
+
line = _ANSI.sub("", line).rstrip()
|
|
109
|
+
if not line or line.startswith("airon: "): # the SDK's own messages are not training output
|
|
110
|
+
return
|
|
111
|
+
if len(self._lines) == self._lines.maxlen:
|
|
112
|
+
self._omitted += 1
|
|
113
|
+
self._lines.append((stream, line[:LINE_LIMIT]))
|
|
114
|
+
|
|
115
|
+
def drain(self) -> list[dict[str, str]]:
|
|
116
|
+
"""Lines captured since the last call, newest 200 at most."""
|
|
117
|
+
with self._lock:
|
|
118
|
+
lines = [{"stream": s, "text": t} for s, t in self._lines]
|
|
119
|
+
omitted, self._omitted = self._omitted, 0
|
|
120
|
+
self._lines.clear()
|
|
121
|
+
if omitted:
|
|
122
|
+
lines.insert(
|
|
123
|
+
0, {"stream": "stderr", "text": f"[airon: {omitted} earlier lines omitted]"}
|
|
124
|
+
)
|
|
125
|
+
return lines
|
airon/_env.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Describe the machine and code a run started on. Best effort: nothing here may raise or stall."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib.metadata
|
|
6
|
+
import os
|
|
7
|
+
import platform
|
|
8
|
+
import re
|
|
9
|
+
import socket
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
import time
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from ._config import env
|
|
16
|
+
from ._system import gpu_inventory
|
|
17
|
+
|
|
18
|
+
# Imported module name -> distribution name, when they differ.
|
|
19
|
+
_KNOWN_LIBS = {
|
|
20
|
+
"torch": "torch",
|
|
21
|
+
"numpy": "numpy",
|
|
22
|
+
"scipy": "scipy",
|
|
23
|
+
"sklearn": "scikit-learn",
|
|
24
|
+
"pandas": "pandas",
|
|
25
|
+
"transformers": "transformers",
|
|
26
|
+
"datasets": "datasets",
|
|
27
|
+
"tokenizers": "tokenizers",
|
|
28
|
+
"accelerate": "accelerate",
|
|
29
|
+
"peft": "peft",
|
|
30
|
+
"trl": "trl",
|
|
31
|
+
"deepspeed": "deepspeed",
|
|
32
|
+
"lightning": "lightning",
|
|
33
|
+
"pytorch_lightning": "pytorch-lightning",
|
|
34
|
+
"jax": "jax",
|
|
35
|
+
"tensorflow": "tensorflow",
|
|
36
|
+
"keras": "keras",
|
|
37
|
+
"triton": "triton",
|
|
38
|
+
"flash_attn": "flash-attn",
|
|
39
|
+
"xformers": "xformers",
|
|
40
|
+
"bitsandbytes": "bitsandbytes",
|
|
41
|
+
"safetensors": "safetensors",
|
|
42
|
+
"diffusers": "diffusers",
|
|
43
|
+
"vllm": "vllm",
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
# Environment variable that proves a provider, and the name we report for it.
|
|
47
|
+
_CLOUDS = (
|
|
48
|
+
("RUNPOD_POD_ID", "runpod"),
|
|
49
|
+
("VAST_CONTAINERID", "vast"),
|
|
50
|
+
("MODAL_TASK_ID", "modal"),
|
|
51
|
+
("PAPERSPACE_FQDN", "paperspace"),
|
|
52
|
+
("KAGGLE_KERNEL_RUN_TYPE", "kaggle"),
|
|
53
|
+
("COLAB_GPU", "colab"),
|
|
54
|
+
("SM_TRAINING_ENV", "sagemaker"),
|
|
55
|
+
("KUBERNETES_SERVICE_HOST", "kubernetes"),
|
|
56
|
+
)
|
|
57
|
+
_URL_CREDENTIALS = re.compile(r"^([a-z][a-z0-9+.-]*://)[^/@]+@", re.IGNORECASE)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def imported_libs() -> list[str]:
|
|
61
|
+
"""Names of well-known libraries the program has already imported (never imports any)."""
|
|
62
|
+
return [name for name in _KNOWN_LIBS if name in sys.modules]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _lib_versions(names: list[str]) -> dict[str, str]:
|
|
66
|
+
libs: dict[str, str] = {}
|
|
67
|
+
for name in names:
|
|
68
|
+
version = getattr(sys.modules.get(name), "__version__", None)
|
|
69
|
+
if not isinstance(version, str):
|
|
70
|
+
try:
|
|
71
|
+
version = importlib.metadata.version(_KNOWN_LIBS[name])
|
|
72
|
+
except Exception:
|
|
73
|
+
continue
|
|
74
|
+
libs[_KNOWN_LIBS[name]] = version
|
|
75
|
+
return libs
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _cloud() -> dict[str, str]:
|
|
79
|
+
override = env("AIRON_CLOUD")
|
|
80
|
+
if override:
|
|
81
|
+
return {"cloud": override[:64]}
|
|
82
|
+
for variable, name in _CLOUDS:
|
|
83
|
+
value = os.environ.get(variable)
|
|
84
|
+
if value is not None:
|
|
85
|
+
info = {"cloud": name}
|
|
86
|
+
if name == "runpod":
|
|
87
|
+
info["cloud_instance"] = value[:128]
|
|
88
|
+
return info
|
|
89
|
+
return {}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class _Budget:
|
|
93
|
+
"""A shared time limit so a slow git or nvidia-smi cannot stall the run's start."""
|
|
94
|
+
|
|
95
|
+
def __init__(self, seconds: float):
|
|
96
|
+
self._end = time.monotonic() + seconds
|
|
97
|
+
|
|
98
|
+
def left(self) -> float:
|
|
99
|
+
return max(0.0, self._end - time.monotonic())
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _git(args: list[str], budget: _Budget) -> str | None:
|
|
103
|
+
if budget.left() <= 0:
|
|
104
|
+
return None
|
|
105
|
+
try:
|
|
106
|
+
done = subprocess.run(
|
|
107
|
+
["git", *args],
|
|
108
|
+
capture_output=True,
|
|
109
|
+
text=True,
|
|
110
|
+
timeout=min(2.0, budget.left()),
|
|
111
|
+
cwd=os.getcwd(),
|
|
112
|
+
env={**os.environ, "GIT_TERMINAL_PROMPT": "0"},
|
|
113
|
+
)
|
|
114
|
+
except Exception:
|
|
115
|
+
return None
|
|
116
|
+
return done.stdout.strip() if done.returncode == 0 else None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _git_info(budget: _Budget) -> dict[str, Any] | None:
|
|
120
|
+
commit = _git(["rev-parse", "--short=12", "HEAD"], budget)
|
|
121
|
+
if not commit:
|
|
122
|
+
return None
|
|
123
|
+
info: dict[str, Any] = {"commit": commit}
|
|
124
|
+
branch = _git(["rev-parse", "--abbrev-ref", "HEAD"], budget)
|
|
125
|
+
if branch:
|
|
126
|
+
info["branch"] = branch
|
|
127
|
+
status = _git(["status", "--porcelain", "--untracked-files=no"], budget)
|
|
128
|
+
if status is not None:
|
|
129
|
+
info["dirty"] = bool(status)
|
|
130
|
+
remote = _git(["config", "--get", "remote.origin.url"], budget)
|
|
131
|
+
if remote:
|
|
132
|
+
# Remotes sometimes embed a token: https://user:token@example.com/...
|
|
133
|
+
info["remote"] = _URL_CREDENTIALS.sub(r"\1", remote)
|
|
134
|
+
return info
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def capture(libs: list[str] | None = None, budget_seconds: float = 3.0) -> dict[str, Any]:
|
|
138
|
+
"""The ``env`` object of the create request."""
|
|
139
|
+
budget = _Budget(budget_seconds)
|
|
140
|
+
info: dict[str, Any] = {}
|
|
141
|
+
for key, read in (
|
|
142
|
+
("python", platform.python_version),
|
|
143
|
+
("platform", platform.platform),
|
|
144
|
+
("host", socket.gethostname),
|
|
145
|
+
):
|
|
146
|
+
try:
|
|
147
|
+
info[key] = read()
|
|
148
|
+
except Exception:
|
|
149
|
+
continue
|
|
150
|
+
info.update(_cloud())
|
|
151
|
+
try:
|
|
152
|
+
gpus = gpu_inventory()
|
|
153
|
+
if gpus:
|
|
154
|
+
info["gpus"] = gpus
|
|
155
|
+
except Exception:
|
|
156
|
+
pass
|
|
157
|
+
git = _git_info(budget)
|
|
158
|
+
if git:
|
|
159
|
+
info["git"] = git
|
|
160
|
+
versions = _lib_versions(imported_libs() if libs is None else libs)
|
|
161
|
+
if versions:
|
|
162
|
+
info["libs"] = versions
|
|
163
|
+
return info
|
airon/_hooks.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Process-wide hooks: unhandled exceptions, normal exit, SIGTERM and fork.
|
|
2
|
+
|
|
3
|
+
Existing handlers are always chained, never replaced: ours run around them.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import atexit
|
|
9
|
+
import os
|
|
10
|
+
import signal
|
|
11
|
+
import sys
|
|
12
|
+
import threading
|
|
13
|
+
from types import FrameType, TracebackType
|
|
14
|
+
from typing import TYPE_CHECKING, Any, Callable
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from ._run import Run
|
|
18
|
+
|
|
19
|
+
_lock = threading.Lock()
|
|
20
|
+
_runs: list[Run] = []
|
|
21
|
+
_installed = False
|
|
22
|
+
_signal_installed = False
|
|
23
|
+
_previous_excepthook: Callable[..., Any] | None = None
|
|
24
|
+
_previous_sigterm: Any = None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def register(run: Run) -> None:
|
|
28
|
+
with _lock:
|
|
29
|
+
_runs.append(run)
|
|
30
|
+
_install()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def unregister(run: Run) -> None:
|
|
34
|
+
with _lock:
|
|
35
|
+
if run in _runs:
|
|
36
|
+
_runs.remove(run)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def active() -> list[Run]:
|
|
40
|
+
with _lock:
|
|
41
|
+
return list(_runs)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _install() -> None:
|
|
45
|
+
global _installed, _previous_excepthook
|
|
46
|
+
with _lock:
|
|
47
|
+
first = not _installed
|
|
48
|
+
_installed = True
|
|
49
|
+
if first:
|
|
50
|
+
_previous_excepthook = sys.excepthook
|
|
51
|
+
sys.excepthook = _excepthook
|
|
52
|
+
atexit.register(_at_exit)
|
|
53
|
+
if hasattr(os, "register_at_fork"):
|
|
54
|
+
os.register_at_fork(after_in_child=_after_fork)
|
|
55
|
+
_install_sigterm()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _excepthook(
|
|
59
|
+
exc_type: type[BaseException], exc: BaseException, tb: TracebackType | None
|
|
60
|
+
) -> None:
|
|
61
|
+
previous = _previous_excepthook or sys.__excepthook__
|
|
62
|
+
try:
|
|
63
|
+
previous(exc_type, exc, tb) # print the traceback before we spend time flushing
|
|
64
|
+
finally:
|
|
65
|
+
for run in active():
|
|
66
|
+
run._finish_from_exception(exc_type, exc, tb)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _at_exit() -> None:
|
|
70
|
+
# If another hook replaced ours, Python still records the unhandled exception here.
|
|
71
|
+
last = getattr(sys, "last_value", None)
|
|
72
|
+
for run in active():
|
|
73
|
+
if last is not None:
|
|
74
|
+
run._finish_from_exception(type(last), last, last.__traceback__)
|
|
75
|
+
else:
|
|
76
|
+
run._finish_at_exit()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _after_fork() -> None:
|
|
80
|
+
# A forked child shares the run object but not its threads; it must not report for the parent.
|
|
81
|
+
for run in active():
|
|
82
|
+
run._disable_in_child()
|
|
83
|
+
with _lock:
|
|
84
|
+
_runs.clear()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _install_sigterm() -> None:
|
|
88
|
+
global _signal_installed, _previous_sigterm
|
|
89
|
+
if _signal_installed or threading.current_thread() is not threading.main_thread():
|
|
90
|
+
return
|
|
91
|
+
try:
|
|
92
|
+
previous = signal.getsignal(signal.SIGTERM)
|
|
93
|
+
if previous is None or previous == signal.SIG_IGN:
|
|
94
|
+
return # installed outside Python, or deliberately ignored: leave it alone
|
|
95
|
+
_previous_sigterm = previous
|
|
96
|
+
signal.signal(signal.SIGTERM, _on_sigterm)
|
|
97
|
+
_signal_installed = True
|
|
98
|
+
except (ValueError, OSError):
|
|
99
|
+
return
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _on_sigterm(signum: int, frame: FrameType | None) -> None:
|
|
103
|
+
previous = _previous_sigterm
|
|
104
|
+
final = not callable(previous)
|
|
105
|
+
try:
|
|
106
|
+
if callable(previous):
|
|
107
|
+
previous(signum, frame) # the program's own handler goes first
|
|
108
|
+
except BaseException:
|
|
109
|
+
final = True # the handler is ending the program (SystemExit and the like)
|
|
110
|
+
raise
|
|
111
|
+
finally:
|
|
112
|
+
for run in active():
|
|
113
|
+
run._finish_from_signal(final)
|
|
114
|
+
if not callable(previous):
|
|
115
|
+
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
|
116
|
+
os.kill(os.getpid(), signal.SIGTERM)
|
airon/_log.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Terminal messages. Everything the SDK says goes through here so keys can be redacted."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import sys
|
|
7
|
+
import threading
|
|
8
|
+
|
|
9
|
+
_lock = threading.Lock()
|
|
10
|
+
_seen: set[str] = set()
|
|
11
|
+
_secrets: list[str] = []
|
|
12
|
+
_BEARER = re.compile(r"(?i)(bearer\s+)\S+")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def add_secret(secret: str | None) -> None:
|
|
16
|
+
# Very short strings would also blank out ordinary words in messages.
|
|
17
|
+
if secret and len(secret) >= 6 and secret not in _secrets:
|
|
18
|
+
_secrets.append(secret)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def redact(text: str) -> str:
|
|
22
|
+
for secret in _secrets:
|
|
23
|
+
text = text.replace(secret, "***")
|
|
24
|
+
return _BEARER.sub(r"\1***", text)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def reset() -> None:
|
|
28
|
+
"""Forget which warnings were printed, so a new run can warn again."""
|
|
29
|
+
with _lock:
|
|
30
|
+
_seen.clear()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _write(message: str) -> None:
|
|
34
|
+
try:
|
|
35
|
+
print(redact(message), file=sys.stderr, flush=True)
|
|
36
|
+
except Exception:
|
|
37
|
+
pass # a closed or broken stderr must never reach the training code
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def warn_once(kind: str, message: str) -> None:
|
|
41
|
+
"""Print a warning, at most once per kind of problem."""
|
|
42
|
+
with _lock:
|
|
43
|
+
if kind in _seen:
|
|
44
|
+
return
|
|
45
|
+
_seen.add(kind)
|
|
46
|
+
_write(f"airon: warning: {message}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def say(message: str) -> None:
|
|
50
|
+
"""Print an informational line unless AIRON_QUIET is set."""
|
|
51
|
+
from ._config import env_flag
|
|
52
|
+
|
|
53
|
+
if not env_flag("AIRON_QUIET"):
|
|
54
|
+
_write(f"airon: {message}")
|