airon 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
airon/__init__.py ADDED
@@ -0,0 +1,30 @@
1
+ """Airon: track machine learning runs without getting in the way of training.
2
+
3
+ import airon
4
+
5
+ run = airon.init(project="my-project", total_steps=50_000)
6
+ run.log({"train/loss": 0.4}, step=100)
7
+ run.finish()
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from typing import TYPE_CHECKING, Any
13
+
14
+ from . import _run
15
+ from ._run import NoopRun as NoopRun
16
+ from ._run import Run as Run
17
+ from ._run import finish as finish
18
+ from ._run import init as init
19
+ from ._run import log as log
20
+ from ._version import __version__ as __version__
21
+
22
+ if TYPE_CHECKING:
23
+ run: Run | None
24
+
25
+
26
+ def __getattr__(name: str) -> Any:
27
+ # ``airon.run`` is the current run, like ``wandb.run``; it changes on init and finish.
28
+ if name == "run":
29
+ return _run.current()
30
+ raise AttributeError(f"module 'airon' has no attribute {name!r}")
airon/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
airon/_config.py ADDED
@@ -0,0 +1,138 @@
1
+ """Where settings come from: arguments, then environment variables, then the credentials file."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import tempfile
8
+ from pathlib import Path
9
+ from typing import Any
10
+ from urllib.parse import urlsplit
11
+
12
+ MODES = ("online", "offline", "disabled")
13
+ RESUME_MODES = ("never", "allow", "must")
14
+
15
+
16
+ def env(name: str) -> str | None:
17
+ value = os.environ.get(name)
18
+ return value.strip() if value and value.strip() else None
19
+
20
+
21
+ def env_flag(name: str) -> bool:
22
+ return (env(name) or "").lower() in ("1", "true", "yes", "on")
23
+
24
+
25
+ def _home() -> Path:
26
+ try:
27
+ return Path.home()
28
+ except Exception:
29
+ return Path(tempfile.gettempdir())
30
+
31
+
32
+ def airon_dir() -> Path | None:
33
+ value = env("AIRON_DIR")
34
+ return Path(value).expanduser() if value else None
35
+
36
+
37
+ def spool_root() -> Path:
38
+ """Directory holding one sub-directory per spooled run."""
39
+ base = airon_dir() or (_home() / ".cache" / "airon")
40
+ return base / "runs"
41
+
42
+
43
+ def credentials_path() -> Path:
44
+ base = airon_dir() or (_home() / ".config" / "airon")
45
+ return base / "credentials.json"
46
+
47
+
48
+ def load_credentials() -> dict[str, str]:
49
+ try:
50
+ data = json.loads(credentials_path().read_text(encoding="utf-8"))
51
+ except Exception:
52
+ return {}
53
+ if not isinstance(data, dict):
54
+ return {}
55
+ return {k: v for k, v in data.items() if isinstance(v, str)}
56
+
57
+
58
+ def save_credentials(url: str, api_key: str) -> Path:
59
+ """Write the credentials file readable by its owner only."""
60
+ path = credentials_path()
61
+ created = not path.parent.exists()
62
+ path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
63
+ if created:
64
+ os.chmod(path.parent, 0o700)
65
+ tmp = path.with_name(path.name + ".tmp")
66
+ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
67
+ with os.fdopen(fd, "w", encoding="utf-8") as handle:
68
+ json.dump({"url": url, "api_key": api_key}, handle)
69
+ os.chmod(tmp, 0o600)
70
+ os.replace(tmp, path)
71
+ return path
72
+
73
+
74
+ def delete_credentials() -> bool:
75
+ try:
76
+ credentials_path().unlink()
77
+ except FileNotFoundError:
78
+ return False
79
+ return True
80
+
81
+
82
+ def normalize_url(raw: str | None) -> str | None:
83
+ """Return ``scheme://host[:port][/prefix]`` without a trailing slash, or None if unusable."""
84
+ if not raw:
85
+ return None
86
+ text = raw.strip()
87
+ if "://" not in text:
88
+ text = "https://" + text
89
+ try:
90
+ parts = urlsplit(text)
91
+ if parts.scheme not in ("http", "https") or not parts.hostname:
92
+ return None
93
+ parts.port # noqa: B018 - raises ValueError for a malformed port
94
+ except ValueError:
95
+ return None
96
+ return f"{parts.scheme}://{parts.netloc}{parts.path}".rstrip("/")
97
+
98
+
99
+ class Connection:
100
+ """The resolved server address and key, and where each came from."""
101
+
102
+ def __init__(
103
+ self,
104
+ url: str | None,
105
+ api_key: str | None,
106
+ url_source: str,
107
+ key_source: str,
108
+ raw_url: str | None,
109
+ ):
110
+ self.url = url
111
+ self.raw_url = raw_url
112
+ self.api_key = api_key
113
+ self.url_source = url_source
114
+ self.key_source = key_source
115
+
116
+ @property
117
+ def complete(self) -> bool:
118
+ return bool(self.url and self.api_key)
119
+
120
+
121
+ def resolve_connection(url: str | None = None, api_key: str | None = None) -> Connection:
122
+ stored: dict[str, Any] | None = None
123
+
124
+ def pick(argument: str | None, variable: str, field: str) -> tuple[str | None, str]:
125
+ nonlocal stored
126
+ if argument:
127
+ return argument, "argument"
128
+ if (value := env(variable)) is not None:
129
+ return value, f"environment variable {variable}"
130
+ if stored is None:
131
+ stored = load_credentials()
132
+ if stored.get(field):
133
+ return stored[field], str(credentials_path())
134
+ return None, "not set"
135
+
136
+ raw_url, url_source = pick(url, "AIRON_URL", "url")
137
+ key, key_source = pick(api_key, "AIRON_API_KEY", "api_key")
138
+ return Connection(normalize_url(raw_url), key, url_source, key_source, raw_url)
airon/_console.py ADDED
@@ -0,0 +1,125 @@
1
+ """Capture the tail of stdout and stderr without changing what the program prints."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ import sys
7
+ import threading
8
+ from collections import deque
9
+ from typing import Any
10
+
11
+ LINE_LIMIT = 4000
12
+ _SPLIT = re.compile(r"(\r\n|\n|\r)")
13
+ _ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]")
14
+
15
+
16
+ class _Tee:
17
+ """Forwards everything to the real stream and shows the text to the capture."""
18
+
19
+ def __init__(self, original: Any, name: str, capture: ConsoleCapture):
20
+ self._original = original
21
+ self._name = name
22
+ self._capture = capture
23
+
24
+ def write(self, text: str) -> int:
25
+ written = self._original.write(text)
26
+ try:
27
+ self._capture.feed(self._name, text)
28
+ except Exception:
29
+ pass # capturing must never turn into an error in the program's print()
30
+ return len(text) if written is None else int(written)
31
+
32
+ def writelines(self, lines: Any) -> None:
33
+ for line in lines:
34
+ self.write(line)
35
+
36
+ def __getattr__(self, name: str) -> Any:
37
+ return getattr(self._original, name)
38
+
39
+ def __iter__(self) -> Any:
40
+ return iter(self._original)
41
+
42
+
43
+ class _Assembler:
44
+ """Builds lines from a stream of text, treating a lone carriage return as "start over"."""
45
+
46
+ def __init__(self) -> None:
47
+ self.parts: list[str] = []
48
+ self.overwrite = False
49
+
50
+ def feed(self, text: str) -> list[str]:
51
+ done: list[str] = []
52
+ for piece in _SPLIT.split(text):
53
+ if piece in ("\n", "\r\n"):
54
+ done.append("".join(self.parts))
55
+ self.parts = []
56
+ self.overwrite = False
57
+ elif piece == "\r":
58
+ self.overwrite = True
59
+ elif piece:
60
+ if self.overwrite:
61
+ self.parts = []
62
+ self.overwrite = False
63
+ self.parts.append(piece)
64
+ return done
65
+
66
+ def rest(self) -> str:
67
+ text = "".join(self.parts)
68
+ self.parts = []
69
+ return text
70
+
71
+
72
+ class ConsoleCapture:
73
+ def __init__(self, max_lines: int = 200):
74
+ self._lock = threading.RLock()
75
+ self._lines: deque[tuple[str, str]] = deque(maxlen=max(1, max_lines))
76
+ self._assemblers = {"stdout": _Assembler(), "stderr": _Assembler()}
77
+ self._omitted = 0
78
+ self._tees: dict[str, _Tee] = {}
79
+ self._originals: dict[str, Any] = {}
80
+ self._active = False
81
+
82
+ def install(self) -> None:
83
+ for name in ("stdout", "stderr"):
84
+ original = getattr(sys, name)
85
+ self._originals[name] = original
86
+ tee = _Tee(original, name, self)
87
+ self._tees[name] = tee
88
+ setattr(sys, name, tee)
89
+ self._active = True
90
+
91
+ def uninstall(self) -> None:
92
+ for name, tee in self._tees.items():
93
+ if getattr(sys, name) is tee:
94
+ setattr(sys, name, self._originals[name])
95
+ with self._lock:
96
+ self._active = False
97
+ for name, assembler in self._assemblers.items():
98
+ self._add(name, assembler.rest())
99
+
100
+ def feed(self, stream: str, text: str) -> None:
101
+ if not self._active:
102
+ return
103
+ with self._lock:
104
+ for line in self._assemblers[stream].feed(text):
105
+ self._add(stream, line)
106
+
107
+ def _add(self, stream: str, line: str) -> None:
108
+ line = _ANSI.sub("", line).rstrip()
109
+ if not line or line.startswith("airon: "): # the SDK's own messages are not training output
110
+ return
111
+ if len(self._lines) == self._lines.maxlen:
112
+ self._omitted += 1
113
+ self._lines.append((stream, line[:LINE_LIMIT]))
114
+
115
+ def drain(self) -> list[dict[str, str]]:
116
+ """Lines captured since the last call, newest 200 at most."""
117
+ with self._lock:
118
+ lines = [{"stream": s, "text": t} for s, t in self._lines]
119
+ omitted, self._omitted = self._omitted, 0
120
+ self._lines.clear()
121
+ if omitted:
122
+ lines.insert(
123
+ 0, {"stream": "stderr", "text": f"[airon: {omitted} earlier lines omitted]"}
124
+ )
125
+ return lines
airon/_env.py ADDED
@@ -0,0 +1,163 @@
1
+ """Describe the machine and code a run started on. Best effort: nothing here may raise or stall."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import importlib.metadata
6
+ import os
7
+ import platform
8
+ import re
9
+ import socket
10
+ import subprocess
11
+ import sys
12
+ import time
13
+ from typing import Any
14
+
15
+ from ._config import env
16
+ from ._system import gpu_inventory
17
+
18
+ # Imported module name -> distribution name, when they differ.
19
+ _KNOWN_LIBS = {
20
+ "torch": "torch",
21
+ "numpy": "numpy",
22
+ "scipy": "scipy",
23
+ "sklearn": "scikit-learn",
24
+ "pandas": "pandas",
25
+ "transformers": "transformers",
26
+ "datasets": "datasets",
27
+ "tokenizers": "tokenizers",
28
+ "accelerate": "accelerate",
29
+ "peft": "peft",
30
+ "trl": "trl",
31
+ "deepspeed": "deepspeed",
32
+ "lightning": "lightning",
33
+ "pytorch_lightning": "pytorch-lightning",
34
+ "jax": "jax",
35
+ "tensorflow": "tensorflow",
36
+ "keras": "keras",
37
+ "triton": "triton",
38
+ "flash_attn": "flash-attn",
39
+ "xformers": "xformers",
40
+ "bitsandbytes": "bitsandbytes",
41
+ "safetensors": "safetensors",
42
+ "diffusers": "diffusers",
43
+ "vllm": "vllm",
44
+ }
45
+
46
+ # Environment variable that proves a provider, and the name we report for it.
47
+ _CLOUDS = (
48
+ ("RUNPOD_POD_ID", "runpod"),
49
+ ("VAST_CONTAINERID", "vast"),
50
+ ("MODAL_TASK_ID", "modal"),
51
+ ("PAPERSPACE_FQDN", "paperspace"),
52
+ ("KAGGLE_KERNEL_RUN_TYPE", "kaggle"),
53
+ ("COLAB_GPU", "colab"),
54
+ ("SM_TRAINING_ENV", "sagemaker"),
55
+ ("KUBERNETES_SERVICE_HOST", "kubernetes"),
56
+ )
57
+ _URL_CREDENTIALS = re.compile(r"^([a-z][a-z0-9+.-]*://)[^/@]+@", re.IGNORECASE)
58
+
59
+
60
+ def imported_libs() -> list[str]:
61
+ """Names of well-known libraries the program has already imported (never imports any)."""
62
+ return [name for name in _KNOWN_LIBS if name in sys.modules]
63
+
64
+
65
+ def _lib_versions(names: list[str]) -> dict[str, str]:
66
+ libs: dict[str, str] = {}
67
+ for name in names:
68
+ version = getattr(sys.modules.get(name), "__version__", None)
69
+ if not isinstance(version, str):
70
+ try:
71
+ version = importlib.metadata.version(_KNOWN_LIBS[name])
72
+ except Exception:
73
+ continue
74
+ libs[_KNOWN_LIBS[name]] = version
75
+ return libs
76
+
77
+
78
+ def _cloud() -> dict[str, str]:
79
+ override = env("AIRON_CLOUD")
80
+ if override:
81
+ return {"cloud": override[:64]}
82
+ for variable, name in _CLOUDS:
83
+ value = os.environ.get(variable)
84
+ if value is not None:
85
+ info = {"cloud": name}
86
+ if name == "runpod":
87
+ info["cloud_instance"] = value[:128]
88
+ return info
89
+ return {}
90
+
91
+
92
+ class _Budget:
93
+ """A shared time limit so a slow git or nvidia-smi cannot stall the run's start."""
94
+
95
+ def __init__(self, seconds: float):
96
+ self._end = time.monotonic() + seconds
97
+
98
+ def left(self) -> float:
99
+ return max(0.0, self._end - time.monotonic())
100
+
101
+
102
+ def _git(args: list[str], budget: _Budget) -> str | None:
103
+ if budget.left() <= 0:
104
+ return None
105
+ try:
106
+ done = subprocess.run(
107
+ ["git", *args],
108
+ capture_output=True,
109
+ text=True,
110
+ timeout=min(2.0, budget.left()),
111
+ cwd=os.getcwd(),
112
+ env={**os.environ, "GIT_TERMINAL_PROMPT": "0"},
113
+ )
114
+ except Exception:
115
+ return None
116
+ return done.stdout.strip() if done.returncode == 0 else None
117
+
118
+
119
+ def _git_info(budget: _Budget) -> dict[str, Any] | None:
120
+ commit = _git(["rev-parse", "--short=12", "HEAD"], budget)
121
+ if not commit:
122
+ return None
123
+ info: dict[str, Any] = {"commit": commit}
124
+ branch = _git(["rev-parse", "--abbrev-ref", "HEAD"], budget)
125
+ if branch:
126
+ info["branch"] = branch
127
+ status = _git(["status", "--porcelain", "--untracked-files=no"], budget)
128
+ if status is not None:
129
+ info["dirty"] = bool(status)
130
+ remote = _git(["config", "--get", "remote.origin.url"], budget)
131
+ if remote:
132
+ # Remotes sometimes embed a token: https://user:token@example.com/...
133
+ info["remote"] = _URL_CREDENTIALS.sub(r"\1", remote)
134
+ return info
135
+
136
+
137
+ def capture(libs: list[str] | None = None, budget_seconds: float = 3.0) -> dict[str, Any]:
138
+ """The ``env`` object of the create request."""
139
+ budget = _Budget(budget_seconds)
140
+ info: dict[str, Any] = {}
141
+ for key, read in (
142
+ ("python", platform.python_version),
143
+ ("platform", platform.platform),
144
+ ("host", socket.gethostname),
145
+ ):
146
+ try:
147
+ info[key] = read()
148
+ except Exception:
149
+ continue
150
+ info.update(_cloud())
151
+ try:
152
+ gpus = gpu_inventory()
153
+ if gpus:
154
+ info["gpus"] = gpus
155
+ except Exception:
156
+ pass
157
+ git = _git_info(budget)
158
+ if git:
159
+ info["git"] = git
160
+ versions = _lib_versions(imported_libs() if libs is None else libs)
161
+ if versions:
162
+ info["libs"] = versions
163
+ return info
airon/_hooks.py ADDED
@@ -0,0 +1,116 @@
1
+ """Process-wide hooks: unhandled exceptions, normal exit, SIGTERM and fork.
2
+
3
+ Existing handlers are always chained, never replaced: ours run around them.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import atexit
9
+ import os
10
+ import signal
11
+ import sys
12
+ import threading
13
+ from types import FrameType, TracebackType
14
+ from typing import TYPE_CHECKING, Any, Callable
15
+
16
+ if TYPE_CHECKING:
17
+ from ._run import Run
18
+
19
+ _lock = threading.Lock()
20
+ _runs: list[Run] = []
21
+ _installed = False
22
+ _signal_installed = False
23
+ _previous_excepthook: Callable[..., Any] | None = None
24
+ _previous_sigterm: Any = None
25
+
26
+
27
+ def register(run: Run) -> None:
28
+ with _lock:
29
+ _runs.append(run)
30
+ _install()
31
+
32
+
33
+ def unregister(run: Run) -> None:
34
+ with _lock:
35
+ if run in _runs:
36
+ _runs.remove(run)
37
+
38
+
39
+ def active() -> list[Run]:
40
+ with _lock:
41
+ return list(_runs)
42
+
43
+
44
+ def _install() -> None:
45
+ global _installed, _previous_excepthook
46
+ with _lock:
47
+ first = not _installed
48
+ _installed = True
49
+ if first:
50
+ _previous_excepthook = sys.excepthook
51
+ sys.excepthook = _excepthook
52
+ atexit.register(_at_exit)
53
+ if hasattr(os, "register_at_fork"):
54
+ os.register_at_fork(after_in_child=_after_fork)
55
+ _install_sigterm()
56
+
57
+
58
+ def _excepthook(
59
+ exc_type: type[BaseException], exc: BaseException, tb: TracebackType | None
60
+ ) -> None:
61
+ previous = _previous_excepthook or sys.__excepthook__
62
+ try:
63
+ previous(exc_type, exc, tb) # print the traceback before we spend time flushing
64
+ finally:
65
+ for run in active():
66
+ run._finish_from_exception(exc_type, exc, tb)
67
+
68
+
69
+ def _at_exit() -> None:
70
+ # If another hook replaced ours, Python still records the unhandled exception here.
71
+ last = getattr(sys, "last_value", None)
72
+ for run in active():
73
+ if last is not None:
74
+ run._finish_from_exception(type(last), last, last.__traceback__)
75
+ else:
76
+ run._finish_at_exit()
77
+
78
+
79
+ def _after_fork() -> None:
80
+ # A forked child shares the run object but not its threads; it must not report for the parent.
81
+ for run in active():
82
+ run._disable_in_child()
83
+ with _lock:
84
+ _runs.clear()
85
+
86
+
87
+ def _install_sigterm() -> None:
88
+ global _signal_installed, _previous_sigterm
89
+ if _signal_installed or threading.current_thread() is not threading.main_thread():
90
+ return
91
+ try:
92
+ previous = signal.getsignal(signal.SIGTERM)
93
+ if previous is None or previous == signal.SIG_IGN:
94
+ return # installed outside Python, or deliberately ignored: leave it alone
95
+ _previous_sigterm = previous
96
+ signal.signal(signal.SIGTERM, _on_sigterm)
97
+ _signal_installed = True
98
+ except (ValueError, OSError):
99
+ return
100
+
101
+
102
+ def _on_sigterm(signum: int, frame: FrameType | None) -> None:
103
+ previous = _previous_sigterm
104
+ final = not callable(previous)
105
+ try:
106
+ if callable(previous):
107
+ previous(signum, frame) # the program's own handler goes first
108
+ except BaseException:
109
+ final = True # the handler is ending the program (SystemExit and the like)
110
+ raise
111
+ finally:
112
+ for run in active():
113
+ run._finish_from_signal(final)
114
+ if not callable(previous):
115
+ signal.signal(signal.SIGTERM, signal.SIG_DFL)
116
+ os.kill(os.getpid(), signal.SIGTERM)
airon/_log.py ADDED
@@ -0,0 +1,54 @@
1
+ """Terminal messages. Everything the SDK says goes through here so keys can be redacted."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ import sys
7
+ import threading
8
+
9
+ _lock = threading.Lock()
10
+ _seen: set[str] = set()
11
+ _secrets: list[str] = []
12
+ _BEARER = re.compile(r"(?i)(bearer\s+)\S+")
13
+
14
+
15
+ def add_secret(secret: str | None) -> None:
16
+ # Very short strings would also blank out ordinary words in messages.
17
+ if secret and len(secret) >= 6 and secret not in _secrets:
18
+ _secrets.append(secret)
19
+
20
+
21
+ def redact(text: str) -> str:
22
+ for secret in _secrets:
23
+ text = text.replace(secret, "***")
24
+ return _BEARER.sub(r"\1***", text)
25
+
26
+
27
+ def reset() -> None:
28
+ """Forget which warnings were printed, so a new run can warn again."""
29
+ with _lock:
30
+ _seen.clear()
31
+
32
+
33
+ def _write(message: str) -> None:
34
+ try:
35
+ print(redact(message), file=sys.stderr, flush=True)
36
+ except Exception:
37
+ pass # a closed or broken stderr must never reach the training code
38
+
39
+
40
+ def warn_once(kind: str, message: str) -> None:
41
+ """Print a warning, at most once per kind of problem."""
42
+ with _lock:
43
+ if kind in _seen:
44
+ return
45
+ _seen.add(kind)
46
+ _write(f"airon: warning: {message}")
47
+
48
+
49
+ def say(message: str) -> None:
50
+ """Print an informational line unless AIRON_QUIET is set."""
51
+ from ._config import env_flag
52
+
53
+ if not env_flag("AIRON_QUIET"):
54
+ _write(f"airon: {message}")