authtracker 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,310 @@
1
+ """MLflow backend. Requires: pip install 'authtracker[mlflow]'."""
2
+ from __future__ import annotations
3
+
4
+ import contextlib
5
+ import importlib
6
+ import importlib.util
7
+ import logging
8
+ from typing import Any, Dict
9
+
10
+ import mlflow
11
+ from mlflow.tracking import MlflowClient
12
+
13
+ from ..errors import BackendError
14
+ from ..settings import Settings
15
+ from .base import Backend, TAG_MODEL_URI, TAG_RUN_ID
16
+
17
+ _FLAVOR_BY_MODULE = {
18
+ "sklearn": "sklearn", "torch": "pytorch", "xgboost": "xgboost",
19
+ "lightgbm": "lightgbm", "transformers": "transformers",
20
+ "tensorflow": "tensorflow", "keras": "keras",
21
+ }
22
+
23
+
24
+ def _detect_flavor(model: Any) -> str:
25
+ """Pick the MLflow saver from the model's class or any of its parents.
26
+
27
+ Looking at the parents matters: your own `class MyNet(nn.Module)` is defined in your code, but
28
+ its parent comes from torch, so it is a PyTorch model. The class closest to your own wins
29
+ (an `XGBClassifier` is xgboost, even though it also is a scikit-learn estimator).
30
+ """
31
+ for cls in type(model).__mro__:
32
+ top = (cls.__module__ or "").split(".")[0]
33
+ if top in _FLAVOR_BY_MODULE:
34
+ return _FLAVOR_BY_MODULE[top]
35
+ raise BackendError(
36
+ f"Cannot detect an MLflow flavor for {type(model).__name__}; "
37
+ "pass flavor='sklearn'|'pytorch'|... to save()"
38
+ )
39
+
40
+
41
+ @contextlib.contextmanager
42
+ def _on_cpu_while_saving(model: Any, flavor: str):
43
+ """A PyTorch model that was trained on a GPU is saved from its CPU weights.
44
+
45
+ MLflow checks the model with a CPU example input while saving, which fails if the model sits on
46
+ a GPU (`cuda` or `mps`). So the model is moved to the CPU for the save and put back afterwards,
47
+ in place, so the training code (and its optimizer) carries on as before.
48
+ """
49
+ try:
50
+ import torch
51
+ except ImportError:
52
+ yield
53
+ return
54
+ if flavor != "pytorch" or not isinstance(model, torch.nn.Module):
55
+ yield
56
+ return
57
+ devices = {t.device for t in list(model.parameters()) + list(model.buffers())}
58
+ if len(devices) != 1 or next(iter(devices)).type == "cpu":
59
+ yield # already on the CPU, or spread over several devices
60
+ return
61
+ original = next(iter(devices))
62
+ model.to("cpu")
63
+ try:
64
+ yield
65
+ finally:
66
+ model.to(original)
67
+
68
+
69
+ def _pytorch_save_hint(error: Exception, flavor: str, kwargs: Dict[str, Any]):
70
+ """A plain-English message for the two ways saving a PyTorch model usually goes wrong.
71
+
72
+ MLflow saves PyTorch models as a traced graph (`pt2`) by default. That needs an example input,
73
+ and the model must be traceable. Both failures come with long, technical messages.
74
+ """
75
+ if flavor != "pytorch" or "serialization_format" in kwargs:
76
+ return None
77
+ text = str(error)
78
+ if "input_example is required" in text:
79
+ return ("Saving a PyTorch model needs an example input, so it can be traced:\n"
80
+ " at.save(net, input_example=batch.cpu().numpy())\n"
81
+ "If the model cannot be traced, save it with serialization_format='pickle' instead "
82
+ "and ship the file that defines your model class: code_paths=['net.py'].")
83
+ if type(error).__module__.split(".")[0] == "torch" or "pt2" in text:
84
+ first = text.strip().splitlines()[0][:200] if text.strip() else type(error).__name__
85
+ return (f"This model could not be traced with torch.export ({type(error).__name__}: {first}).\n"
86
+ "Typical causes: an `if` that depends on tensor values, or an operation that is not "
87
+ "supported. Save it as a pickle instead, and ship the file that defines your model "
88
+ "class so it can be loaded elsewhere:\n"
89
+ " at.save(net, input_example=batch.cpu().numpy(), serialization_format='pickle', "
90
+ "code_paths=['net.py'])\n"
91
+ "Pickled models can run code when loaded, so only load models you trust.")
92
+ return None
93
+
94
+
95
+ def _quote(v: str) -> str:
96
+ return "'" + str(v).replace("'", "\\'") + "'"
97
+
98
+
99
+ _DATABASE_SCHEMES = ("sqlite", "postgresql", "mysql", "mssql")
100
+
101
+
102
+ def _check_store_support(uri) -> None:
103
+ """A database URI needs the full MLflow; the light `mlflow-skinny` can only talk to a server."""
104
+ if uri and uri.split(":", 1)[0].split("+", 1)[0] in _DATABASE_SCHEMES \
105
+ and importlib.util.find_spec("sqlalchemy") is None:
106
+ raise BackendError(
107
+ f"The tracking URI {uri!r} is a database, which needs the full MLflow, but only "
108
+ "mlflow-skinny is installed. Point AUTHTRACKER_TRACKING_URI at an MLflow server "
109
+ "(http://...), or install the full package: pip install 'authtracker[mlflow]'")
110
+
111
+
112
+ class MlflowBackend(Backend):
113
+ _warned_psutil = False
114
+
115
+ def __init__(self, settings: Settings):
116
+ _check_store_support(settings.tracking_uri or settings.registry_uri)
117
+ if settings.tracking_uri:
118
+ mlflow.set_tracking_uri(settings.tracking_uri)
119
+ registry = settings.registry_uri or settings.tracking_uri
120
+ if registry:
121
+ mlflow.set_registry_uri(registry)
122
+ self._settings = settings
123
+ self.client = MlflowClient()
124
+
125
+ def ping(self) -> None:
126
+ self.client.search_experiments(max_results=1)
127
+
128
+ _MARKERS = ("Max retries exceeded", "Connection refused", "Connection aborted",
129
+ "Failed to establish a new connection", "timed out", "Bad Gateway",
130
+ "Service Unavailable", "Gateway Time-out")
131
+
132
+ def is_transient(self, exc: Exception) -> bool:
133
+ if super().is_transient(exc):
134
+ return True
135
+ if isinstance(exc, mlflow.exceptions.MlflowException):
136
+ return (exc.error_code == "TEMPORARILY_UNAVAILABLE"
137
+ or any(m in str(exc) for m in self._MARKERS))
138
+ return False
139
+
140
+ # ---- runs (fluent API for start/end so nested runs and log_model work)
141
+ def start_run(self, experiment, run_name, tags, parent_id):
142
+ if parent_id is None:
143
+ mlflow.set_experiment(experiment)
144
+ run = mlflow.start_run(run_name=run_name, nested=parent_id is not None, tags=tags,
145
+ **self._system_metrics_args())
146
+ return run.info.run_id
147
+
148
+ def _system_metrics_args(self) -> Dict[str, Any]:
149
+ """CPU / memory / disk / network (and GPU) charts, if asked for and possible.
150
+
151
+ MLflow creates the run first and only then fails when `psutil` is missing, which would
152
+ crash the training script and leave a run stuck as "running". So check before starting.
153
+ """
154
+ if not self._settings.system_metrics:
155
+ return {"log_system_metrics": False} # explicit, so MLflow's own environment variable cannot turn it on
156
+ if importlib.util.find_spec("psutil") is None:
157
+ if not MlflowBackend._warned_psutil:
158
+ MlflowBackend._warned_psutil = True
159
+ logging.getLogger("authtracker").warning(
160
+ "System metrics are enabled but `psutil` is not installed, so they are "
161
+ "skipped. Install it with: pip install psutil (or set AUTHTRACKER_SYSTEM_METRICS=false)")
162
+ return {"log_system_metrics": False}
163
+ return {"log_system_metrics": True}
164
+
165
+ def download_artifact(self, run_id, name, folder):
166
+ if name not in [a.path for a in self.client.list_artifacts(run_id)]:
167
+ return None
168
+ return self.client.download_artifacts(run_id, name, folder)
169
+
170
+ def find_run(self, experiment, name, tags, parent_id):
171
+ exp = self.client.get_experiment_by_name(experiment)
172
+ if exp is None:
173
+ return None
174
+ parts = [f"tags.`mlflow.runName` = {_quote(name)}"] + [f"tags.`{k}` = {_quote(v)}" for k, v in tags.items()]
175
+ if parent_id:
176
+ parts.append(f"tags.`mlflow.parentRunId` = {_quote(parent_id)}")
177
+ runs = self.client.search_runs([exp.experiment_id], " and ".join(parts),
178
+ order_by=["attributes.start_time DESC"], max_results=1)
179
+ return runs[0].info.run_id if runs else None
180
+
181
+ def resume_run(self, run_id, parent_id):
182
+ run = mlflow.start_run(run_id=run_id, nested=parent_id is not None, **self._system_metrics_args())
183
+ return run.info.run_id
184
+
185
+ def last_step(self, run_id):
186
+ steps = [m.step for m in getattr(self.client.get_run(run_id).data, "_metric_objs", [])]
187
+ return max(steps) if steps else None
188
+
189
+ def end_run(self, run_id, status):
190
+ active = mlflow.active_run()
191
+ if active is None or active.info.run_id != run_id:
192
+ self.client.set_terminated(run_id, status) # not the innermost run
193
+ else:
194
+ mlflow.end_run(status=status)
195
+
196
+ def set_tags(self, run_id, tags):
197
+ for k, v in tags.items():
198
+ self.client.set_tag(run_id, k, str(v))
199
+
200
+ def log_params(self, run_id, params):
201
+ for k, v in params.items():
202
+ self.client.log_param(run_id, k, v)
203
+
204
+ def log_metrics(self, run_id, metrics, step):
205
+ for k, v in metrics.items():
206
+ self.client.log_metric(run_id, k, float(v), step=step or 0)
207
+
208
+ def log_artifact(self, run_id, path):
209
+ self.client.log_artifact(run_id, path)
210
+
211
+ def log_model(self, run_id, model, flavor, **kwargs):
212
+ active = mlflow.active_run()
213
+ if active is None or active.info.run_id != run_id:
214
+ raise BackendError(
215
+ "log_model must be called on the innermost open run "
216
+ "(call it inside the `with` block of the run that owns the model)"
217
+ )
218
+ flavor = flavor or _detect_flavor(model)
219
+ module = importlib.import_module(f"mlflow.{flavor}")
220
+ # pyfunc takes the model as `python_model=`; other flavors take it positionally.
221
+ args, kw = ((), {"python_model": model}) if flavor == "pyfunc" else ((model,), {})
222
+ try:
223
+ with _on_cpu_while_saving(model, flavor):
224
+ try:
225
+ info = module.log_model(*args, name="model", **kw, **kwargs)
226
+ except TypeError: # older MLflow
227
+ info = module.log_model(*args, artifact_path="model", **kw, **kwargs)
228
+ except Exception as e:
229
+ hint = _pytorch_save_hint(e, flavor, kwargs)
230
+ if hint:
231
+ raise BackendError(hint) from e
232
+ raise
233
+ uri = info.model_uri
234
+ self.client.set_tag(run_id, TAG_MODEL_URI, uri)
235
+ return uri
236
+
237
+ def _summary(self, run) -> Dict[str, Any]:
238
+ return {
239
+ "run_id": run.info.run_id,
240
+ "name": run.data.tags.get("mlflow.runName"),
241
+ "metrics": dict(run.data.metrics),
242
+ "params": dict(run.data.params),
243
+ "tags": dict(run.data.tags),
244
+ }
245
+
246
+ def get_run(self, run_id):
247
+ return self._summary(self.client.get_run(run_id))
248
+
249
+ def search_runs(self, experiment, tags, metric, mode, limit):
250
+ exp = self.client.get_experiment_by_name(experiment)
251
+ if exp is None:
252
+ return []
253
+ flt = " and ".join(f"tags.`{k}` = {_quote(v)}" for k, v in tags.items())
254
+ flt = f"{flt} and metrics.`{metric}` >= -1e308" if flt else f"metrics.`{metric}` >= -1e308"
255
+ order = f"metrics.`{metric}` {'DESC' if mode == 'max' else 'ASC'}"
256
+ runs = self.client.search_runs([exp.experiment_id], filter_string=flt,
257
+ order_by=[order], max_results=limit)
258
+ return [self._summary(r) for r in runs]
259
+
260
+ def log_model_dir(self, run_id, local_dir):
261
+ """A folder kept as it is, inside an MLflow 'logged model' (so it can be registered)."""
262
+ from .. import bundle
263
+ exp_id = self.client.get_run(run_id).info.experiment_id
264
+ lm = self.client.create_logged_model(experiment_id=exp_id, name="model", source_run_id=run_id,
265
+ model_type=bundle.MODEL_TYPE)
266
+ try:
267
+ self.client.log_model_artifacts(lm.model_id, local_dir)
268
+ except Exception:
269
+ self.client.finalize_logged_model(lm.model_id, "FAILED")
270
+ raise
271
+ self.client.finalize_logged_model(lm.model_id, "READY")
272
+ uri = f"models:/{lm.model_id}"
273
+ self.client.set_tag(run_id, TAG_MODEL_URI, uri)
274
+ return uri
275
+
276
+ def download_bundle(self, name, alias, dst=None):
277
+ from .. import bundle
278
+ try:
279
+ source = self.client.get_model_version_by_alias(name, alias).source
280
+ except mlflow.exceptions.MlflowException:
281
+ return None
282
+ if not source.startswith("models:/m-"):
283
+ return None
284
+ if self.client.get_logged_model(source.split("/", 1)[1]).model_type != bundle.MODEL_TYPE:
285
+ return None
286
+ import tempfile
287
+ return mlflow.artifacts.download_artifacts(
288
+ artifact_uri=f"models:/{name}@{alias}", dst_path=dst or tempfile.mkdtemp(prefix="authtracker-load-"))
289
+
290
+ # ---- registry
291
+ def register(self, run_id, name, model_uri):
292
+ mv = mlflow.register_model(model_uri, name)
293
+ self.client.set_model_version_tag(name, mv.version, TAG_RUN_ID, run_id)
294
+ return str(mv.version)
295
+
296
+ def set_alias(self, name, version, alias):
297
+ self.client.set_registered_model_alias(name, alias, version)
298
+
299
+ def get_alias(self, name, alias):
300
+ try:
301
+ return str(self.client.get_model_version_by_alias(name, alias).version)
302
+ except mlflow.exceptions.MlflowException:
303
+ return None
304
+
305
+ def version_run_id(self, name, version):
306
+ mv = self.client.get_model_version(name, str(version))
307
+ return mv.tags.get(TAG_RUN_ID) or mv.run_id
308
+
309
+ def load(self, name, alias):
310
+ return mlflow.pyfunc.load_model(f"models:/{name}@{alias}")
@@ -0,0 +1,329 @@
1
+ """Retries and an on-disk offline queue around any backend.
2
+
3
+ A tracking outage must not kill a training job:
4
+
5
+ * Every call is retried with exponential backoff when the failure looks transient.
6
+ * Write operations (start/end run, tags, params, metrics, artifacts) that still fail are
7
+ appended to a queue directory and replayed, in order, once the server is back.
8
+ * Operations that need an answer or a live model (log_model, register, search, load...)
9
+ cannot be queued; they raise BackendError after the retries.
10
+
11
+ Queued files are replayed automatically by the process that wrote them (when the server
12
+ answers again, and at interpreter exit). Leftovers from crashed processes are uploaded
13
+ with ``authtracker sync``.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import atexit
18
+ import json
19
+ import logging
20
+ import os
21
+ import shutil
22
+ import time
23
+ import uuid
24
+ from pathlib import Path
25
+ from typing import Any, Callable, Dict, List, Optional
26
+
27
+ from ..errors import BackendError
28
+ from ..settings import Settings
29
+ from .base import Backend
30
+
31
+ log = logging.getLogger("authtracker")
32
+
33
+ OFFLINE = "offline-" # prefix of run ids handed out while the server is unreachable
34
+
35
+
36
+ class _Unreachable(Exception):
37
+ """Transient failures persisted through all retries."""
38
+
39
+
40
+ class _Unresolved(Exception):
41
+ """A queued op refers to an offline run id that was never created."""
42
+
43
+
44
+ class ResilientBackend(Backend):
45
+ def __init__(self, inner: Backend, settings: Settings, *,
46
+ sleep: Callable[[float], None] = time.sleep, cooldown: float = 30.0):
47
+ self.inner = inner
48
+ self.retries = settings.retries
49
+ self.wait = settings.retry_wait
50
+ self.dir = Path(settings.queue_dir
51
+ or Path.home() / ".authtracker" / "queue").expanduser()
52
+ (self.dir / "files").mkdir(parents=True, exist_ok=True)
53
+ (self.dir / "failed").mkdir(exist_ok=True)
54
+ self.token = f"{os.getpid()}-{uuid.uuid4().hex[:8]}"
55
+ self._sleep = sleep
56
+ self._cooldown = cooldown
57
+ self._down_until = 0.0
58
+ self._warned = False
59
+ self._atexit_registered = False
60
+ self._idmap_file = self.dir / "idmap.json"
61
+ self._idmap: Dict[str, str] = self._read_idmap()
62
+
63
+ @property
64
+ def captures_console(self) -> bool: # type: ignore[override]
65
+ return bool(getattr(self.inner, "captures_console", False))
66
+
67
+ def __getattr__(self, name: str) -> Any: # delegate backend-specific attributes
68
+ if name == "inner":
69
+ raise AttributeError(name)
70
+ return getattr(self.inner, name)
71
+
72
+ # ------------------------------------------------------------------ helpers
73
+ def _transient(self, exc: Exception) -> bool:
74
+ return self.inner.is_transient(exc)
75
+
76
+ def _retry(self, fn: Callable[[], Any]) -> Any:
77
+ attempt = 0
78
+ while True:
79
+ try:
80
+ return fn()
81
+ except Exception as e:
82
+ if not self._transient(e):
83
+ raise
84
+ if attempt >= self.retries:
85
+ raise _Unreachable(e) from e
86
+ self._sleep(self.wait * (2 ** attempt))
87
+ attempt += 1
88
+
89
+ def _mark_down(self) -> None:
90
+ self._down_until = time.monotonic() + self._cooldown
91
+
92
+ def _read_idmap(self) -> Dict[str, str]:
93
+ try:
94
+ return json.loads(self._idmap_file.read_text())
95
+ except (FileNotFoundError, ValueError):
96
+ return {}
97
+
98
+ def _save_idmap(self) -> None:
99
+ self._idmap_file.write_text(json.dumps(self._idmap))
100
+
101
+ def _res(self, run_id: Optional[str]) -> Optional[str]:
102
+ return self._idmap.get(run_id, run_id) if run_id else run_id
103
+
104
+ def _unresolved(self, run_id: Optional[str]) -> bool:
105
+ r = self._res(run_id)
106
+ return bool(r) and r.startswith(OFFLINE)
107
+
108
+ # ------------------------------------------------------------------ queue
109
+ def _entries(self, all_tokens: bool) -> List[Path]:
110
+ files = sorted(self.dir.glob("*.json"))
111
+ files = [p for p in files if p.name not in ("idmap.json",)]
112
+ if not all_tokens:
113
+ files = [p for p in files if p.name.endswith(f"-{self.token}.json")]
114
+ return files
115
+
116
+ def pending(self) -> int:
117
+ """Number of queued operations (from every process) waiting to be uploaded."""
118
+ return len(self._entries(all_tokens=True))
119
+
120
+ def _enqueue(self, op: str, args: Dict[str, Any]) -> None:
121
+ path = self.dir / f"{time.time_ns():020d}-{self.token}.json"
122
+ path.write_text(json.dumps({"op": op, "args": args}, default=str))
123
+ if not self._warned:
124
+ self._warned = True
125
+ log.warning("Tracking server unreachable: queuing data in %s "
126
+ "(it is uploaded automatically, or run `authtracker sync`)", self.dir)
127
+ if not self._atexit_registered:
128
+ self._atexit_registered = True
129
+ atexit.register(self._flush_at_exit)
130
+
131
+ def _flush_at_exit(self) -> None:
132
+ try:
133
+ self.flush(force=True)
134
+ left = len(self._entries(all_tokens=False))
135
+ if left:
136
+ log.warning("%d tracking operations remain queued in %s; "
137
+ "run `authtracker sync` later", left, self.dir)
138
+ except Exception:
139
+ pass
140
+
141
+ def _fail(self, path: Path, exc: Exception) -> None:
142
+ log.warning("Dropping queued op %s: %s", path.name, exc)
143
+ shutil.move(str(path), str(self.dir / "failed" / path.name))
144
+
145
+ def _apply(self, entry: Dict[str, Any]) -> None:
146
+ op, a = entry["op"], entry["args"]
147
+ rid = self._res(a.get("run_id"))
148
+ if rid is not None and rid.startswith(OFFLINE):
149
+ raise _Unresolved(rid)
150
+ if op == "start_run":
151
+ parent = self._res(a["parent_id"])
152
+ if parent is not None and parent.startswith(OFFLINE):
153
+ raise _Unresolved(parent)
154
+ real = self.inner.start_run(a["experiment"], a["run_name"], a["tags"], parent)
155
+ self._idmap[a["placeholder"]] = real
156
+ self._save_idmap()
157
+ elif op == "end_run":
158
+ self.inner.end_run(rid, a["status"])
159
+ elif op == "set_tags":
160
+ self.inner.set_tags(rid, a["tags"])
161
+ elif op == "log_params":
162
+ self.inner.log_params(rid, a["params"])
163
+ elif op == "log_metrics":
164
+ self.inner.log_metrics(rid, a["metrics"], a["step"])
165
+ elif op == "log_artifact":
166
+ self.inner.log_artifact(rid, a["path"])
167
+ else:
168
+ raise BackendError(f"Unknown queued op {op!r}")
169
+
170
+ def flush(self, *, force: bool = False, all_tokens: bool = False) -> int:
171
+ """Replay queued operations in order. Returns how many were uploaded."""
172
+ files = self._entries(all_tokens)
173
+ if not files:
174
+ return 0
175
+ if not force and time.monotonic() < self._down_until:
176
+ return 0
177
+ done = 0
178
+ for path in files:
179
+ entry = json.loads(path.read_text())
180
+ try:
181
+ self._apply(entry)
182
+ except _Unresolved as e:
183
+ self._fail(path, e)
184
+ continue
185
+ except Exception as e:
186
+ if self._transient(e):
187
+ self._mark_down()
188
+ break
189
+ self._fail(path, e)
190
+ continue
191
+ if entry["op"] == "log_artifact":
192
+ shutil.rmtree(Path(entry["args"]["path"]).parent, ignore_errors=True)
193
+ path.unlink()
194
+ done += 1
195
+ if done:
196
+ log.info("Uploaded %d queued tracking operations", done)
197
+ return done
198
+
199
+ # ------------------------------------------------------------------ queued writes
200
+ def _write(self, op: str, args: Dict[str, Any], live: Callable[[], Any],
201
+ run_id: Optional[str]) -> None:
202
+ self.flush()
203
+ if self._entries(False) or self._unresolved(run_id):
204
+ self._enqueue(op, args)
205
+ return
206
+ try:
207
+ self._retry(live)
208
+ except _Unreachable:
209
+ self._mark_down()
210
+ self._enqueue(op, args)
211
+
212
+ def start_run(self, experiment, run_name, tags, parent_id):
213
+ self.flush()
214
+
215
+ def enqueue() -> str:
216
+ placeholder = OFFLINE + uuid.uuid4().hex
217
+ self._enqueue("start_run", {
218
+ "experiment": experiment, "run_name": run_name, "tags": tags,
219
+ "parent_id": parent_id, "placeholder": placeholder})
220
+ return placeholder
221
+
222
+ if self._entries(False) or self._unresolved(parent_id):
223
+ return enqueue()
224
+ try:
225
+ return self._retry(lambda: self.inner.start_run(
226
+ experiment, run_name, tags, self._res(parent_id)))
227
+ except _Unreachable:
228
+ self._mark_down()
229
+ return enqueue()
230
+
231
+ def end_run(self, run_id, status):
232
+ self._write("end_run", {"run_id": run_id, "status": status},
233
+ lambda: self.inner.end_run(self._res(run_id), status), run_id)
234
+
235
+ def set_tags(self, run_id, tags):
236
+ self._write("set_tags", {"run_id": run_id, "tags": tags},
237
+ lambda: self.inner.set_tags(self._res(run_id), tags), run_id)
238
+
239
+ def log_params(self, run_id, params):
240
+ self._write("log_params", {"run_id": run_id, "params": params},
241
+ lambda: self.inner.log_params(self._res(run_id), params), run_id)
242
+
243
+ def log_metrics(self, run_id, metrics, step):
244
+ self._write("log_metrics", {"run_id": run_id, "metrics": metrics, "step": step},
245
+ lambda: self.inner.log_metrics(self._res(run_id), metrics, step), run_id)
246
+
247
+ def log_artifact(self, run_id, path):
248
+ # Copy now: the caller may delete the file (e.g. a temp directory) before replay.
249
+ def queue_copy() -> None:
250
+ folder = self.dir / "files" / str(time.time_ns()) # keeps the file's real name
251
+ folder.mkdir(parents=True)
252
+ dest = folder / Path(path).name
253
+ shutil.copy(path, dest)
254
+ self._enqueue("log_artifact", {"run_id": run_id, "path": str(dest)})
255
+
256
+ self.flush()
257
+ if self._entries(False) or self._unresolved(run_id):
258
+ return queue_copy()
259
+ try:
260
+ self._retry(lambda: self.inner.log_artifact(self._res(run_id), path))
261
+ except _Unreachable:
262
+ self._mark_down()
263
+ queue_copy()
264
+
265
+ # ------------------------------------------------------------------ not queueable
266
+ def _read(self, what: str, fn: Callable[[], Any], run_id: Optional[str] = None) -> Any:
267
+ self.flush()
268
+ if self._unresolved(run_id):
269
+ raise BackendError(f"{what}: run {run_id} is not on the server yet "
270
+ "(still queued offline)")
271
+ try:
272
+ return self._retry(fn)
273
+ except _Unreachable as e:
274
+ raise BackendError(
275
+ f"{what}: tracking server unreachable after {self.retries} retries "
276
+ f"({e.__cause__})") from e.__cause__
277
+
278
+ def ping(self) -> None:
279
+ self.inner.ping()
280
+
281
+ def log_model(self, run_id, model, flavor, **kwargs):
282
+ return self._read("log_model",
283
+ lambda: self.inner.log_model(self._res(run_id), model, flavor, **kwargs),
284
+ run_id)
285
+
286
+ def log_model_dir(self, run_id, local_dir):
287
+ return self._read("log_model_dir",
288
+ lambda: self.inner.log_model_dir(self._res(run_id), local_dir), run_id)
289
+
290
+ def download_bundle(self, name, alias, dst=None):
291
+ return self._read("download_bundle", lambda: self.inner.download_bundle(name, alias, dst))
292
+
293
+ def download_artifact(self, run_id, name, folder):
294
+ return self._read("download_artifact", lambda: self.inner.download_artifact(self._res(run_id), name, folder),
295
+ run_id)
296
+
297
+ def find_run(self, experiment, name, tags, parent_id):
298
+ return self._read("find_run", lambda: self.inner.find_run(experiment, name, tags, self._res(parent_id)),
299
+ parent_id)
300
+
301
+ def resume_run(self, run_id, parent_id):
302
+ return self._read("resume_run", lambda: self.inner.resume_run(self._res(run_id), self._res(parent_id)),
303
+ run_id)
304
+
305
+ def last_step(self, run_id):
306
+ return self._read("last_step", lambda: self.inner.last_step(self._res(run_id)), run_id)
307
+
308
+ def get_run(self, run_id):
309
+ return self._read("get_run", lambda: self.inner.get_run(self._res(run_id)), run_id)
310
+
311
+ def search_runs(self, experiment, tags, metric, mode, limit):
312
+ return self._read("search_runs",
313
+ lambda: self.inner.search_runs(experiment, tags, metric, mode, limit))
314
+
315
+ def register(self, run_id, name, model_uri):
316
+ return self._read("register",
317
+ lambda: self.inner.register(self._res(run_id), name, model_uri), run_id)
318
+
319
+ def set_alias(self, name, version, alias):
320
+ return self._read("set_alias", lambda: self.inner.set_alias(name, version, alias))
321
+
322
+ def get_alias(self, name, alias):
323
+ return self._read("get_alias", lambda: self.inner.get_alias(name, alias))
324
+
325
+ def version_run_id(self, name, version):
326
+ return self._read("version_run_id", lambda: self.inner.version_run_id(name, version))
327
+
328
+ def load(self, name, alias):
329
+ return self._read("load", lambda: self.inner.load(name, alias))