authtracker 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- authtracker/__init__.py +16 -0
- authtracker/backends/__init__.py +95 -0
- authtracker/backends/base.py +136 -0
- authtracker/backends/local.py +200 -0
- authtracker/backends/mlflow_backend.py +310 -0
- authtracker/backends/resilient.py +329 -0
- authtracker/backends/wandb_backend.py +235 -0
- authtracker/bundle.py +887 -0
- authtracker/capture.py +175 -0
- authtracker/cli.py +51 -0
- authtracker/configfile.py +84 -0
- authtracker/dataset.py +98 -0
- authtracker/errors.py +18 -0
- authtracker/registry.py +118 -0
- authtracker/run.py +346 -0
- authtracker/settings.py +171 -0
- authtracker/simple.py +241 -0
- authtracker/testing.py +82 -0
- authtracker/utils.py +321 -0
- authtracker/watch.py +59 -0
- authtracker-0.0.1.dist-info/METADATA +215 -0
- authtracker-0.0.1.dist-info/RECORD +25 -0
- authtracker-0.0.1.dist-info/WHEEL +4 -0
- authtracker-0.0.1.dist-info/entry_points.txt +2 -0
- authtracker-0.0.1.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
"""MLflow backend. Requires: pip install 'authtracker[mlflow]'."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import contextlib
|
|
5
|
+
import importlib
|
|
6
|
+
import importlib.util
|
|
7
|
+
import logging
|
|
8
|
+
from typing import Any, Dict
|
|
9
|
+
|
|
10
|
+
import mlflow
|
|
11
|
+
from mlflow.tracking import MlflowClient
|
|
12
|
+
|
|
13
|
+
from ..errors import BackendError
|
|
14
|
+
from ..settings import Settings
|
|
15
|
+
from .base import Backend, TAG_MODEL_URI, TAG_RUN_ID
|
|
16
|
+
|
|
17
|
+
_FLAVOR_BY_MODULE = {
|
|
18
|
+
"sklearn": "sklearn", "torch": "pytorch", "xgboost": "xgboost",
|
|
19
|
+
"lightgbm": "lightgbm", "transformers": "transformers",
|
|
20
|
+
"tensorflow": "tensorflow", "keras": "keras",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _detect_flavor(model: Any) -> str:
|
|
25
|
+
"""Pick the MLflow saver from the model's class or any of its parents.
|
|
26
|
+
|
|
27
|
+
Looking at the parents matters: your own `class MyNet(nn.Module)` is defined in your code, but
|
|
28
|
+
its parent comes from torch, so it is a PyTorch model. The class closest to your own wins
|
|
29
|
+
(an `XGBClassifier` is xgboost, even though it also is a scikit-learn estimator).
|
|
30
|
+
"""
|
|
31
|
+
for cls in type(model).__mro__:
|
|
32
|
+
top = (cls.__module__ or "").split(".")[0]
|
|
33
|
+
if top in _FLAVOR_BY_MODULE:
|
|
34
|
+
return _FLAVOR_BY_MODULE[top]
|
|
35
|
+
raise BackendError(
|
|
36
|
+
f"Cannot detect an MLflow flavor for {type(model).__name__}; "
|
|
37
|
+
"pass flavor='sklearn'|'pytorch'|... to save()"
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@contextlib.contextmanager
|
|
42
|
+
def _on_cpu_while_saving(model: Any, flavor: str):
|
|
43
|
+
"""A PyTorch model that was trained on a GPU is saved from its CPU weights.
|
|
44
|
+
|
|
45
|
+
MLflow checks the model with a CPU example input while saving, which fails if the model sits on
|
|
46
|
+
a GPU (`cuda` or `mps`). So the model is moved to the CPU for the save and put back afterwards,
|
|
47
|
+
in place, so the training code (and its optimizer) carries on as before.
|
|
48
|
+
"""
|
|
49
|
+
try:
|
|
50
|
+
import torch
|
|
51
|
+
except ImportError:
|
|
52
|
+
yield
|
|
53
|
+
return
|
|
54
|
+
if flavor != "pytorch" or not isinstance(model, torch.nn.Module):
|
|
55
|
+
yield
|
|
56
|
+
return
|
|
57
|
+
devices = {t.device for t in list(model.parameters()) + list(model.buffers())}
|
|
58
|
+
if len(devices) != 1 or next(iter(devices)).type == "cpu":
|
|
59
|
+
yield # already on the CPU, or spread over several devices
|
|
60
|
+
return
|
|
61
|
+
original = next(iter(devices))
|
|
62
|
+
model.to("cpu")
|
|
63
|
+
try:
|
|
64
|
+
yield
|
|
65
|
+
finally:
|
|
66
|
+
model.to(original)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _pytorch_save_hint(error: Exception, flavor: str, kwargs: Dict[str, Any]):
|
|
70
|
+
"""A plain-English message for the two ways saving a PyTorch model usually goes wrong.
|
|
71
|
+
|
|
72
|
+
MLflow saves PyTorch models as a traced graph (`pt2`) by default. That needs an example input,
|
|
73
|
+
and the model must be traceable. Both failures come with long, technical messages.
|
|
74
|
+
"""
|
|
75
|
+
if flavor != "pytorch" or "serialization_format" in kwargs:
|
|
76
|
+
return None
|
|
77
|
+
text = str(error)
|
|
78
|
+
if "input_example is required" in text:
|
|
79
|
+
return ("Saving a PyTorch model needs an example input, so it can be traced:\n"
|
|
80
|
+
" at.save(net, input_example=batch.cpu().numpy())\n"
|
|
81
|
+
"If the model cannot be traced, save it with serialization_format='pickle' instead "
|
|
82
|
+
"and ship the file that defines your model class: code_paths=['net.py'].")
|
|
83
|
+
if type(error).__module__.split(".")[0] == "torch" or "pt2" in text:
|
|
84
|
+
first = text.strip().splitlines()[0][:200] if text.strip() else type(error).__name__
|
|
85
|
+
return (f"This model could not be traced with torch.export ({type(error).__name__}: {first}).\n"
|
|
86
|
+
"Typical causes: an `if` that depends on tensor values, or an operation that is not "
|
|
87
|
+
"supported. Save it as a pickle instead, and ship the file that defines your model "
|
|
88
|
+
"class so it can be loaded elsewhere:\n"
|
|
89
|
+
" at.save(net, input_example=batch.cpu().numpy(), serialization_format='pickle', "
|
|
90
|
+
"code_paths=['net.py'])\n"
|
|
91
|
+
"Pickled models can run code when loaded, so only load models you trust.")
|
|
92
|
+
return None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _quote(v: str) -> str:
|
|
96
|
+
return "'" + str(v).replace("'", "\\'") + "'"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
_DATABASE_SCHEMES = ("sqlite", "postgresql", "mysql", "mssql")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _check_store_support(uri) -> None:
|
|
103
|
+
"""A database URI needs the full MLflow; the light `mlflow-skinny` can only talk to a server."""
|
|
104
|
+
if uri and uri.split(":", 1)[0].split("+", 1)[0] in _DATABASE_SCHEMES \
|
|
105
|
+
and importlib.util.find_spec("sqlalchemy") is None:
|
|
106
|
+
raise BackendError(
|
|
107
|
+
f"The tracking URI {uri!r} is a database, which needs the full MLflow, but only "
|
|
108
|
+
"mlflow-skinny is installed. Point AUTHTRACKER_TRACKING_URI at an MLflow server "
|
|
109
|
+
"(http://...), or install the full package: pip install 'authtracker[mlflow]'")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class MlflowBackend(Backend):
|
|
113
|
+
_warned_psutil = False
|
|
114
|
+
|
|
115
|
+
def __init__(self, settings: Settings):
|
|
116
|
+
_check_store_support(settings.tracking_uri or settings.registry_uri)
|
|
117
|
+
if settings.tracking_uri:
|
|
118
|
+
mlflow.set_tracking_uri(settings.tracking_uri)
|
|
119
|
+
registry = settings.registry_uri or settings.tracking_uri
|
|
120
|
+
if registry:
|
|
121
|
+
mlflow.set_registry_uri(registry)
|
|
122
|
+
self._settings = settings
|
|
123
|
+
self.client = MlflowClient()
|
|
124
|
+
|
|
125
|
+
def ping(self) -> None:
|
|
126
|
+
self.client.search_experiments(max_results=1)
|
|
127
|
+
|
|
128
|
+
_MARKERS = ("Max retries exceeded", "Connection refused", "Connection aborted",
|
|
129
|
+
"Failed to establish a new connection", "timed out", "Bad Gateway",
|
|
130
|
+
"Service Unavailable", "Gateway Time-out")
|
|
131
|
+
|
|
132
|
+
def is_transient(self, exc: Exception) -> bool:
|
|
133
|
+
if super().is_transient(exc):
|
|
134
|
+
return True
|
|
135
|
+
if isinstance(exc, mlflow.exceptions.MlflowException):
|
|
136
|
+
return (exc.error_code == "TEMPORARILY_UNAVAILABLE"
|
|
137
|
+
or any(m in str(exc) for m in self._MARKERS))
|
|
138
|
+
return False
|
|
139
|
+
|
|
140
|
+
# ---- runs (fluent API for start/end so nested runs and log_model work)
|
|
141
|
+
def start_run(self, experiment, run_name, tags, parent_id):
|
|
142
|
+
if parent_id is None:
|
|
143
|
+
mlflow.set_experiment(experiment)
|
|
144
|
+
run = mlflow.start_run(run_name=run_name, nested=parent_id is not None, tags=tags,
|
|
145
|
+
**self._system_metrics_args())
|
|
146
|
+
return run.info.run_id
|
|
147
|
+
|
|
148
|
+
def _system_metrics_args(self) -> Dict[str, Any]:
|
|
149
|
+
"""CPU / memory / disk / network (and GPU) charts, if asked for and possible.
|
|
150
|
+
|
|
151
|
+
MLflow creates the run first and only then fails when `psutil` is missing, which would
|
|
152
|
+
crash the training script and leave a run stuck as "running". So check before starting.
|
|
153
|
+
"""
|
|
154
|
+
if not self._settings.system_metrics:
|
|
155
|
+
return {"log_system_metrics": False} # explicit, so MLflow's own environment variable cannot turn it on
|
|
156
|
+
if importlib.util.find_spec("psutil") is None:
|
|
157
|
+
if not MlflowBackend._warned_psutil:
|
|
158
|
+
MlflowBackend._warned_psutil = True
|
|
159
|
+
logging.getLogger("authtracker").warning(
|
|
160
|
+
"System metrics are enabled but `psutil` is not installed, so they are "
|
|
161
|
+
"skipped. Install it with: pip install psutil (or set AUTHTRACKER_SYSTEM_METRICS=false)")
|
|
162
|
+
return {"log_system_metrics": False}
|
|
163
|
+
return {"log_system_metrics": True}
|
|
164
|
+
|
|
165
|
+
def download_artifact(self, run_id, name, folder):
|
|
166
|
+
if name not in [a.path for a in self.client.list_artifacts(run_id)]:
|
|
167
|
+
return None
|
|
168
|
+
return self.client.download_artifacts(run_id, name, folder)
|
|
169
|
+
|
|
170
|
+
def find_run(self, experiment, name, tags, parent_id):
|
|
171
|
+
exp = self.client.get_experiment_by_name(experiment)
|
|
172
|
+
if exp is None:
|
|
173
|
+
return None
|
|
174
|
+
parts = [f"tags.`mlflow.runName` = {_quote(name)}"] + [f"tags.`{k}` = {_quote(v)}" for k, v in tags.items()]
|
|
175
|
+
if parent_id:
|
|
176
|
+
parts.append(f"tags.`mlflow.parentRunId` = {_quote(parent_id)}")
|
|
177
|
+
runs = self.client.search_runs([exp.experiment_id], " and ".join(parts),
|
|
178
|
+
order_by=["attributes.start_time DESC"], max_results=1)
|
|
179
|
+
return runs[0].info.run_id if runs else None
|
|
180
|
+
|
|
181
|
+
def resume_run(self, run_id, parent_id):
|
|
182
|
+
run = mlflow.start_run(run_id=run_id, nested=parent_id is not None, **self._system_metrics_args())
|
|
183
|
+
return run.info.run_id
|
|
184
|
+
|
|
185
|
+
def last_step(self, run_id):
|
|
186
|
+
steps = [m.step for m in getattr(self.client.get_run(run_id).data, "_metric_objs", [])]
|
|
187
|
+
return max(steps) if steps else None
|
|
188
|
+
|
|
189
|
+
def end_run(self, run_id, status):
|
|
190
|
+
active = mlflow.active_run()
|
|
191
|
+
if active is None or active.info.run_id != run_id:
|
|
192
|
+
self.client.set_terminated(run_id, status) # not the innermost run
|
|
193
|
+
else:
|
|
194
|
+
mlflow.end_run(status=status)
|
|
195
|
+
|
|
196
|
+
def set_tags(self, run_id, tags):
|
|
197
|
+
for k, v in tags.items():
|
|
198
|
+
self.client.set_tag(run_id, k, str(v))
|
|
199
|
+
|
|
200
|
+
def log_params(self, run_id, params):
|
|
201
|
+
for k, v in params.items():
|
|
202
|
+
self.client.log_param(run_id, k, v)
|
|
203
|
+
|
|
204
|
+
def log_metrics(self, run_id, metrics, step):
|
|
205
|
+
for k, v in metrics.items():
|
|
206
|
+
self.client.log_metric(run_id, k, float(v), step=step or 0)
|
|
207
|
+
|
|
208
|
+
def log_artifact(self, run_id, path):
|
|
209
|
+
self.client.log_artifact(run_id, path)
|
|
210
|
+
|
|
211
|
+
def log_model(self, run_id, model, flavor, **kwargs):
|
|
212
|
+
active = mlflow.active_run()
|
|
213
|
+
if active is None or active.info.run_id != run_id:
|
|
214
|
+
raise BackendError(
|
|
215
|
+
"log_model must be called on the innermost open run "
|
|
216
|
+
"(call it inside the `with` block of the run that owns the model)"
|
|
217
|
+
)
|
|
218
|
+
flavor = flavor or _detect_flavor(model)
|
|
219
|
+
module = importlib.import_module(f"mlflow.{flavor}")
|
|
220
|
+
# pyfunc takes the model as `python_model=`; other flavors take it positionally.
|
|
221
|
+
args, kw = ((), {"python_model": model}) if flavor == "pyfunc" else ((model,), {})
|
|
222
|
+
try:
|
|
223
|
+
with _on_cpu_while_saving(model, flavor):
|
|
224
|
+
try:
|
|
225
|
+
info = module.log_model(*args, name="model", **kw, **kwargs)
|
|
226
|
+
except TypeError: # older MLflow
|
|
227
|
+
info = module.log_model(*args, artifact_path="model", **kw, **kwargs)
|
|
228
|
+
except Exception as e:
|
|
229
|
+
hint = _pytorch_save_hint(e, flavor, kwargs)
|
|
230
|
+
if hint:
|
|
231
|
+
raise BackendError(hint) from e
|
|
232
|
+
raise
|
|
233
|
+
uri = info.model_uri
|
|
234
|
+
self.client.set_tag(run_id, TAG_MODEL_URI, uri)
|
|
235
|
+
return uri
|
|
236
|
+
|
|
237
|
+
def _summary(self, run) -> Dict[str, Any]:
|
|
238
|
+
return {
|
|
239
|
+
"run_id": run.info.run_id,
|
|
240
|
+
"name": run.data.tags.get("mlflow.runName"),
|
|
241
|
+
"metrics": dict(run.data.metrics),
|
|
242
|
+
"params": dict(run.data.params),
|
|
243
|
+
"tags": dict(run.data.tags),
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
def get_run(self, run_id):
|
|
247
|
+
return self._summary(self.client.get_run(run_id))
|
|
248
|
+
|
|
249
|
+
def search_runs(self, experiment, tags, metric, mode, limit):
|
|
250
|
+
exp = self.client.get_experiment_by_name(experiment)
|
|
251
|
+
if exp is None:
|
|
252
|
+
return []
|
|
253
|
+
flt = " and ".join(f"tags.`{k}` = {_quote(v)}" for k, v in tags.items())
|
|
254
|
+
flt = f"{flt} and metrics.`{metric}` >= -1e308" if flt else f"metrics.`{metric}` >= -1e308"
|
|
255
|
+
order = f"metrics.`{metric}` {'DESC' if mode == 'max' else 'ASC'}"
|
|
256
|
+
runs = self.client.search_runs([exp.experiment_id], filter_string=flt,
|
|
257
|
+
order_by=[order], max_results=limit)
|
|
258
|
+
return [self._summary(r) for r in runs]
|
|
259
|
+
|
|
260
|
+
def log_model_dir(self, run_id, local_dir):
|
|
261
|
+
"""A folder kept as it is, inside an MLflow 'logged model' (so it can be registered)."""
|
|
262
|
+
from .. import bundle
|
|
263
|
+
exp_id = self.client.get_run(run_id).info.experiment_id
|
|
264
|
+
lm = self.client.create_logged_model(experiment_id=exp_id, name="model", source_run_id=run_id,
|
|
265
|
+
model_type=bundle.MODEL_TYPE)
|
|
266
|
+
try:
|
|
267
|
+
self.client.log_model_artifacts(lm.model_id, local_dir)
|
|
268
|
+
except Exception:
|
|
269
|
+
self.client.finalize_logged_model(lm.model_id, "FAILED")
|
|
270
|
+
raise
|
|
271
|
+
self.client.finalize_logged_model(lm.model_id, "READY")
|
|
272
|
+
uri = f"models:/{lm.model_id}"
|
|
273
|
+
self.client.set_tag(run_id, TAG_MODEL_URI, uri)
|
|
274
|
+
return uri
|
|
275
|
+
|
|
276
|
+
def download_bundle(self, name, alias, dst=None):
|
|
277
|
+
from .. import bundle
|
|
278
|
+
try:
|
|
279
|
+
source = self.client.get_model_version_by_alias(name, alias).source
|
|
280
|
+
except mlflow.exceptions.MlflowException:
|
|
281
|
+
return None
|
|
282
|
+
if not source.startswith("models:/m-"):
|
|
283
|
+
return None
|
|
284
|
+
if self.client.get_logged_model(source.split("/", 1)[1]).model_type != bundle.MODEL_TYPE:
|
|
285
|
+
return None
|
|
286
|
+
import tempfile
|
|
287
|
+
return mlflow.artifacts.download_artifacts(
|
|
288
|
+
artifact_uri=f"models:/{name}@{alias}", dst_path=dst or tempfile.mkdtemp(prefix="authtracker-load-"))
|
|
289
|
+
|
|
290
|
+
# ---- registry
|
|
291
|
+
def register(self, run_id, name, model_uri):
|
|
292
|
+
mv = mlflow.register_model(model_uri, name)
|
|
293
|
+
self.client.set_model_version_tag(name, mv.version, TAG_RUN_ID, run_id)
|
|
294
|
+
return str(mv.version)
|
|
295
|
+
|
|
296
|
+
def set_alias(self, name, version, alias):
|
|
297
|
+
self.client.set_registered_model_alias(name, alias, version)
|
|
298
|
+
|
|
299
|
+
def get_alias(self, name, alias):
|
|
300
|
+
try:
|
|
301
|
+
return str(self.client.get_model_version_by_alias(name, alias).version)
|
|
302
|
+
except mlflow.exceptions.MlflowException:
|
|
303
|
+
return None
|
|
304
|
+
|
|
305
|
+
def version_run_id(self, name, version):
|
|
306
|
+
mv = self.client.get_model_version(name, str(version))
|
|
307
|
+
return mv.tags.get(TAG_RUN_ID) or mv.run_id
|
|
308
|
+
|
|
309
|
+
def load(self, name, alias):
|
|
310
|
+
return mlflow.pyfunc.load_model(f"models:/{name}@{alias}")
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
"""Retries and an on-disk offline queue around any backend.
|
|
2
|
+
|
|
3
|
+
A tracking outage must not kill a training job:
|
|
4
|
+
|
|
5
|
+
* Every call is retried with exponential backoff when the failure looks transient.
|
|
6
|
+
* Write operations (start/end run, tags, params, metrics, artifacts) that still fail are
|
|
7
|
+
appended to a queue directory and replayed, in order, once the server is back.
|
|
8
|
+
* Operations that need an answer or a live model (log_model, register, search, load...)
|
|
9
|
+
cannot be queued; they raise BackendError after the retries.
|
|
10
|
+
|
|
11
|
+
Queued files are replayed automatically by the process that wrote them (when the server
|
|
12
|
+
answers again, and at interpreter exit). Leftovers from crashed processes are uploaded
|
|
13
|
+
with ``authtracker sync``.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import atexit
|
|
18
|
+
import json
|
|
19
|
+
import logging
|
|
20
|
+
import os
|
|
21
|
+
import shutil
|
|
22
|
+
import time
|
|
23
|
+
import uuid
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
26
|
+
|
|
27
|
+
from ..errors import BackendError
|
|
28
|
+
from ..settings import Settings
|
|
29
|
+
from .base import Backend
|
|
30
|
+
|
|
31
|
+
log = logging.getLogger("authtracker")
|
|
32
|
+
|
|
33
|
+
OFFLINE = "offline-" # prefix of run ids handed out while the server is unreachable
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class _Unreachable(Exception):
|
|
37
|
+
"""Transient failures persisted through all retries."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class _Unresolved(Exception):
|
|
41
|
+
"""A queued op refers to an offline run id that was never created."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ResilientBackend(Backend):
|
|
45
|
+
def __init__(self, inner: Backend, settings: Settings, *,
|
|
46
|
+
sleep: Callable[[float], None] = time.sleep, cooldown: float = 30.0):
|
|
47
|
+
self.inner = inner
|
|
48
|
+
self.retries = settings.retries
|
|
49
|
+
self.wait = settings.retry_wait
|
|
50
|
+
self.dir = Path(settings.queue_dir
|
|
51
|
+
or Path.home() / ".authtracker" / "queue").expanduser()
|
|
52
|
+
(self.dir / "files").mkdir(parents=True, exist_ok=True)
|
|
53
|
+
(self.dir / "failed").mkdir(exist_ok=True)
|
|
54
|
+
self.token = f"{os.getpid()}-{uuid.uuid4().hex[:8]}"
|
|
55
|
+
self._sleep = sleep
|
|
56
|
+
self._cooldown = cooldown
|
|
57
|
+
self._down_until = 0.0
|
|
58
|
+
self._warned = False
|
|
59
|
+
self._atexit_registered = False
|
|
60
|
+
self._idmap_file = self.dir / "idmap.json"
|
|
61
|
+
self._idmap: Dict[str, str] = self._read_idmap()
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def captures_console(self) -> bool: # type: ignore[override]
|
|
65
|
+
return bool(getattr(self.inner, "captures_console", False))
|
|
66
|
+
|
|
67
|
+
def __getattr__(self, name: str) -> Any: # delegate backend-specific attributes
|
|
68
|
+
if name == "inner":
|
|
69
|
+
raise AttributeError(name)
|
|
70
|
+
return getattr(self.inner, name)
|
|
71
|
+
|
|
72
|
+
# ------------------------------------------------------------------ helpers
|
|
73
|
+
def _transient(self, exc: Exception) -> bool:
|
|
74
|
+
return self.inner.is_transient(exc)
|
|
75
|
+
|
|
76
|
+
def _retry(self, fn: Callable[[], Any]) -> Any:
|
|
77
|
+
attempt = 0
|
|
78
|
+
while True:
|
|
79
|
+
try:
|
|
80
|
+
return fn()
|
|
81
|
+
except Exception as e:
|
|
82
|
+
if not self._transient(e):
|
|
83
|
+
raise
|
|
84
|
+
if attempt >= self.retries:
|
|
85
|
+
raise _Unreachable(e) from e
|
|
86
|
+
self._sleep(self.wait * (2 ** attempt))
|
|
87
|
+
attempt += 1
|
|
88
|
+
|
|
89
|
+
def _mark_down(self) -> None:
|
|
90
|
+
self._down_until = time.monotonic() + self._cooldown
|
|
91
|
+
|
|
92
|
+
def _read_idmap(self) -> Dict[str, str]:
|
|
93
|
+
try:
|
|
94
|
+
return json.loads(self._idmap_file.read_text())
|
|
95
|
+
except (FileNotFoundError, ValueError):
|
|
96
|
+
return {}
|
|
97
|
+
|
|
98
|
+
def _save_idmap(self) -> None:
|
|
99
|
+
self._idmap_file.write_text(json.dumps(self._idmap))
|
|
100
|
+
|
|
101
|
+
def _res(self, run_id: Optional[str]) -> Optional[str]:
|
|
102
|
+
return self._idmap.get(run_id, run_id) if run_id else run_id
|
|
103
|
+
|
|
104
|
+
def _unresolved(self, run_id: Optional[str]) -> bool:
|
|
105
|
+
r = self._res(run_id)
|
|
106
|
+
return bool(r) and r.startswith(OFFLINE)
|
|
107
|
+
|
|
108
|
+
# ------------------------------------------------------------------ queue
|
|
109
|
+
def _entries(self, all_tokens: bool) -> List[Path]:
|
|
110
|
+
files = sorted(self.dir.glob("*.json"))
|
|
111
|
+
files = [p for p in files if p.name not in ("idmap.json",)]
|
|
112
|
+
if not all_tokens:
|
|
113
|
+
files = [p for p in files if p.name.endswith(f"-{self.token}.json")]
|
|
114
|
+
return files
|
|
115
|
+
|
|
116
|
+
def pending(self) -> int:
|
|
117
|
+
"""Number of queued operations (from every process) waiting to be uploaded."""
|
|
118
|
+
return len(self._entries(all_tokens=True))
|
|
119
|
+
|
|
120
|
+
def _enqueue(self, op: str, args: Dict[str, Any]) -> None:
|
|
121
|
+
path = self.dir / f"{time.time_ns():020d}-{self.token}.json"
|
|
122
|
+
path.write_text(json.dumps({"op": op, "args": args}, default=str))
|
|
123
|
+
if not self._warned:
|
|
124
|
+
self._warned = True
|
|
125
|
+
log.warning("Tracking server unreachable: queuing data in %s "
|
|
126
|
+
"(it is uploaded automatically, or run `authtracker sync`)", self.dir)
|
|
127
|
+
if not self._atexit_registered:
|
|
128
|
+
self._atexit_registered = True
|
|
129
|
+
atexit.register(self._flush_at_exit)
|
|
130
|
+
|
|
131
|
+
def _flush_at_exit(self) -> None:
|
|
132
|
+
try:
|
|
133
|
+
self.flush(force=True)
|
|
134
|
+
left = len(self._entries(all_tokens=False))
|
|
135
|
+
if left:
|
|
136
|
+
log.warning("%d tracking operations remain queued in %s; "
|
|
137
|
+
"run `authtracker sync` later", left, self.dir)
|
|
138
|
+
except Exception:
|
|
139
|
+
pass
|
|
140
|
+
|
|
141
|
+
def _fail(self, path: Path, exc: Exception) -> None:
|
|
142
|
+
log.warning("Dropping queued op %s: %s", path.name, exc)
|
|
143
|
+
shutil.move(str(path), str(self.dir / "failed" / path.name))
|
|
144
|
+
|
|
145
|
+
def _apply(self, entry: Dict[str, Any]) -> None:
|
|
146
|
+
op, a = entry["op"], entry["args"]
|
|
147
|
+
rid = self._res(a.get("run_id"))
|
|
148
|
+
if rid is not None and rid.startswith(OFFLINE):
|
|
149
|
+
raise _Unresolved(rid)
|
|
150
|
+
if op == "start_run":
|
|
151
|
+
parent = self._res(a["parent_id"])
|
|
152
|
+
if parent is not None and parent.startswith(OFFLINE):
|
|
153
|
+
raise _Unresolved(parent)
|
|
154
|
+
real = self.inner.start_run(a["experiment"], a["run_name"], a["tags"], parent)
|
|
155
|
+
self._idmap[a["placeholder"]] = real
|
|
156
|
+
self._save_idmap()
|
|
157
|
+
elif op == "end_run":
|
|
158
|
+
self.inner.end_run(rid, a["status"])
|
|
159
|
+
elif op == "set_tags":
|
|
160
|
+
self.inner.set_tags(rid, a["tags"])
|
|
161
|
+
elif op == "log_params":
|
|
162
|
+
self.inner.log_params(rid, a["params"])
|
|
163
|
+
elif op == "log_metrics":
|
|
164
|
+
self.inner.log_metrics(rid, a["metrics"], a["step"])
|
|
165
|
+
elif op == "log_artifact":
|
|
166
|
+
self.inner.log_artifact(rid, a["path"])
|
|
167
|
+
else:
|
|
168
|
+
raise BackendError(f"Unknown queued op {op!r}")
|
|
169
|
+
|
|
170
|
+
def flush(self, *, force: bool = False, all_tokens: bool = False) -> int:
|
|
171
|
+
"""Replay queued operations in order. Returns how many were uploaded."""
|
|
172
|
+
files = self._entries(all_tokens)
|
|
173
|
+
if not files:
|
|
174
|
+
return 0
|
|
175
|
+
if not force and time.monotonic() < self._down_until:
|
|
176
|
+
return 0
|
|
177
|
+
done = 0
|
|
178
|
+
for path in files:
|
|
179
|
+
entry = json.loads(path.read_text())
|
|
180
|
+
try:
|
|
181
|
+
self._apply(entry)
|
|
182
|
+
except _Unresolved as e:
|
|
183
|
+
self._fail(path, e)
|
|
184
|
+
continue
|
|
185
|
+
except Exception as e:
|
|
186
|
+
if self._transient(e):
|
|
187
|
+
self._mark_down()
|
|
188
|
+
break
|
|
189
|
+
self._fail(path, e)
|
|
190
|
+
continue
|
|
191
|
+
if entry["op"] == "log_artifact":
|
|
192
|
+
shutil.rmtree(Path(entry["args"]["path"]).parent, ignore_errors=True)
|
|
193
|
+
path.unlink()
|
|
194
|
+
done += 1
|
|
195
|
+
if done:
|
|
196
|
+
log.info("Uploaded %d queued tracking operations", done)
|
|
197
|
+
return done
|
|
198
|
+
|
|
199
|
+
# ------------------------------------------------------------------ queued writes
|
|
200
|
+
def _write(self, op: str, args: Dict[str, Any], live: Callable[[], Any],
|
|
201
|
+
run_id: Optional[str]) -> None:
|
|
202
|
+
self.flush()
|
|
203
|
+
if self._entries(False) or self._unresolved(run_id):
|
|
204
|
+
self._enqueue(op, args)
|
|
205
|
+
return
|
|
206
|
+
try:
|
|
207
|
+
self._retry(live)
|
|
208
|
+
except _Unreachable:
|
|
209
|
+
self._mark_down()
|
|
210
|
+
self._enqueue(op, args)
|
|
211
|
+
|
|
212
|
+
def start_run(self, experiment, run_name, tags, parent_id):
|
|
213
|
+
self.flush()
|
|
214
|
+
|
|
215
|
+
def enqueue() -> str:
|
|
216
|
+
placeholder = OFFLINE + uuid.uuid4().hex
|
|
217
|
+
self._enqueue("start_run", {
|
|
218
|
+
"experiment": experiment, "run_name": run_name, "tags": tags,
|
|
219
|
+
"parent_id": parent_id, "placeholder": placeholder})
|
|
220
|
+
return placeholder
|
|
221
|
+
|
|
222
|
+
if self._entries(False) or self._unresolved(parent_id):
|
|
223
|
+
return enqueue()
|
|
224
|
+
try:
|
|
225
|
+
return self._retry(lambda: self.inner.start_run(
|
|
226
|
+
experiment, run_name, tags, self._res(parent_id)))
|
|
227
|
+
except _Unreachable:
|
|
228
|
+
self._mark_down()
|
|
229
|
+
return enqueue()
|
|
230
|
+
|
|
231
|
+
def end_run(self, run_id, status):
|
|
232
|
+
self._write("end_run", {"run_id": run_id, "status": status},
|
|
233
|
+
lambda: self.inner.end_run(self._res(run_id), status), run_id)
|
|
234
|
+
|
|
235
|
+
def set_tags(self, run_id, tags):
|
|
236
|
+
self._write("set_tags", {"run_id": run_id, "tags": tags},
|
|
237
|
+
lambda: self.inner.set_tags(self._res(run_id), tags), run_id)
|
|
238
|
+
|
|
239
|
+
def log_params(self, run_id, params):
|
|
240
|
+
self._write("log_params", {"run_id": run_id, "params": params},
|
|
241
|
+
lambda: self.inner.log_params(self._res(run_id), params), run_id)
|
|
242
|
+
|
|
243
|
+
def log_metrics(self, run_id, metrics, step):
|
|
244
|
+
self._write("log_metrics", {"run_id": run_id, "metrics": metrics, "step": step},
|
|
245
|
+
lambda: self.inner.log_metrics(self._res(run_id), metrics, step), run_id)
|
|
246
|
+
|
|
247
|
+
def log_artifact(self, run_id, path):
|
|
248
|
+
# Copy now: the caller may delete the file (e.g. a temp directory) before replay.
|
|
249
|
+
def queue_copy() -> None:
|
|
250
|
+
folder = self.dir / "files" / str(time.time_ns()) # keeps the file's real name
|
|
251
|
+
folder.mkdir(parents=True)
|
|
252
|
+
dest = folder / Path(path).name
|
|
253
|
+
shutil.copy(path, dest)
|
|
254
|
+
self._enqueue("log_artifact", {"run_id": run_id, "path": str(dest)})
|
|
255
|
+
|
|
256
|
+
self.flush()
|
|
257
|
+
if self._entries(False) or self._unresolved(run_id):
|
|
258
|
+
return queue_copy()
|
|
259
|
+
try:
|
|
260
|
+
self._retry(lambda: self.inner.log_artifact(self._res(run_id), path))
|
|
261
|
+
except _Unreachable:
|
|
262
|
+
self._mark_down()
|
|
263
|
+
queue_copy()
|
|
264
|
+
|
|
265
|
+
# ------------------------------------------------------------------ not queueable
|
|
266
|
+
def _read(self, what: str, fn: Callable[[], Any], run_id: Optional[str] = None) -> Any:
|
|
267
|
+
self.flush()
|
|
268
|
+
if self._unresolved(run_id):
|
|
269
|
+
raise BackendError(f"{what}: run {run_id} is not on the server yet "
|
|
270
|
+
"(still queued offline)")
|
|
271
|
+
try:
|
|
272
|
+
return self._retry(fn)
|
|
273
|
+
except _Unreachable as e:
|
|
274
|
+
raise BackendError(
|
|
275
|
+
f"{what}: tracking server unreachable after {self.retries} retries "
|
|
276
|
+
f"({e.__cause__})") from e.__cause__
|
|
277
|
+
|
|
278
|
+
def ping(self) -> None:
|
|
279
|
+
self.inner.ping()
|
|
280
|
+
|
|
281
|
+
def log_model(self, run_id, model, flavor, **kwargs):
|
|
282
|
+
return self._read("log_model",
|
|
283
|
+
lambda: self.inner.log_model(self._res(run_id), model, flavor, **kwargs),
|
|
284
|
+
run_id)
|
|
285
|
+
|
|
286
|
+
def log_model_dir(self, run_id, local_dir):
|
|
287
|
+
return self._read("log_model_dir",
|
|
288
|
+
lambda: self.inner.log_model_dir(self._res(run_id), local_dir), run_id)
|
|
289
|
+
|
|
290
|
+
def download_bundle(self, name, alias, dst=None):
|
|
291
|
+
return self._read("download_bundle", lambda: self.inner.download_bundle(name, alias, dst))
|
|
292
|
+
|
|
293
|
+
def download_artifact(self, run_id, name, folder):
|
|
294
|
+
return self._read("download_artifact", lambda: self.inner.download_artifact(self._res(run_id), name, folder),
|
|
295
|
+
run_id)
|
|
296
|
+
|
|
297
|
+
def find_run(self, experiment, name, tags, parent_id):
|
|
298
|
+
return self._read("find_run", lambda: self.inner.find_run(experiment, name, tags, self._res(parent_id)),
|
|
299
|
+
parent_id)
|
|
300
|
+
|
|
301
|
+
def resume_run(self, run_id, parent_id):
|
|
302
|
+
return self._read("resume_run", lambda: self.inner.resume_run(self._res(run_id), self._res(parent_id)),
|
|
303
|
+
run_id)
|
|
304
|
+
|
|
305
|
+
def last_step(self, run_id):
|
|
306
|
+
return self._read("last_step", lambda: self.inner.last_step(self._res(run_id)), run_id)
|
|
307
|
+
|
|
308
|
+
def get_run(self, run_id):
|
|
309
|
+
return self._read("get_run", lambda: self.inner.get_run(self._res(run_id)), run_id)
|
|
310
|
+
|
|
311
|
+
def search_runs(self, experiment, tags, metric, mode, limit):
|
|
312
|
+
return self._read("search_runs",
|
|
313
|
+
lambda: self.inner.search_runs(experiment, tags, metric, mode, limit))
|
|
314
|
+
|
|
315
|
+
def register(self, run_id, name, model_uri):
|
|
316
|
+
return self._read("register",
|
|
317
|
+
lambda: self.inner.register(self._res(run_id), name, model_uri), run_id)
|
|
318
|
+
|
|
319
|
+
def set_alias(self, name, version, alias):
|
|
320
|
+
return self._read("set_alias", lambda: self.inner.set_alias(name, version, alias))
|
|
321
|
+
|
|
322
|
+
def get_alias(self, name, alias):
|
|
323
|
+
return self._read("get_alias", lambda: self.inner.get_alias(name, alias))
|
|
324
|
+
|
|
325
|
+
def version_run_id(self, name, version):
|
|
326
|
+
return self._read("version_run_id", lambda: self.inner.version_run_id(name, version))
|
|
327
|
+
|
|
328
|
+
def load(self, name, alias):
|
|
329
|
+
return self._read("load", lambda: self.inner.load(name, alias))
|