authtracker 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- authtracker/__init__.py +16 -0
- authtracker/backends/__init__.py +95 -0
- authtracker/backends/base.py +136 -0
- authtracker/backends/local.py +200 -0
- authtracker/backends/mlflow_backend.py +310 -0
- authtracker/backends/resilient.py +329 -0
- authtracker/backends/wandb_backend.py +235 -0
- authtracker/bundle.py +887 -0
- authtracker/capture.py +175 -0
- authtracker/cli.py +51 -0
- authtracker/configfile.py +84 -0
- authtracker/dataset.py +98 -0
- authtracker/errors.py +18 -0
- authtracker/registry.py +118 -0
- authtracker/run.py +346 -0
- authtracker/settings.py +171 -0
- authtracker/simple.py +241 -0
- authtracker/testing.py +82 -0
- authtracker/utils.py +321 -0
- authtracker/watch.py +59 -0
- authtracker-0.0.1.dist-info/METADATA +215 -0
- authtracker-0.0.1.dist-info/RECORD +25 -0
- authtracker-0.0.1.dist-info/WHEEL +4 -0
- authtracker-0.0.1.dist-info/entry_points.txt +2 -0
- authtracker-0.0.1.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""Weights & Biases backend. Requires: pip install wandb
|
|
2
|
+
|
|
3
|
+
Mapping (same three levels as every backend):
|
|
4
|
+
|
|
5
|
+
model -> W&B project
|
|
6
|
+
experiment -> run group; the parent run (job type "experiment")
|
|
7
|
+
run -> a run (job type "run") in that group
|
|
8
|
+
|
|
9
|
+
Settings: sign in with WANDB_API_KEY, optional WANDB_ENTITY (team/user), WANDB_MODE=offline to
|
|
10
|
+
write locally and `wandb sync` later. AUTHTRACKER_TRACKING_URI is used as the W&B base URL for a
|
|
11
|
+
self-hosted server. Run ids look like "<project>/<wandb run id>".
|
|
12
|
+
|
|
13
|
+
Supported: runs, params, metrics, tags, files, best_run, and models saved as artifacts.
|
|
14
|
+
Not supported yet: the model registry (register / promote / rollback / load). Those raise a clear
|
|
15
|
+
BackendError.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import os
|
|
20
|
+
import pickle
|
|
21
|
+
import shutil
|
|
22
|
+
from typing import Any, Dict, List, Optional
|
|
23
|
+
|
|
24
|
+
from ..errors import BackendError
|
|
25
|
+
from ..settings import Settings
|
|
26
|
+
from .base import TAG_LEVEL, TAG_MODEL_URI, Backend
|
|
27
|
+
|
|
28
|
+
_TAGS_KEY = "authtracker_tags" # our key/value tags, stored in the run config
|
|
29
|
+
_PARENT_KEY = "authtracker_parent"
|
|
30
|
+
_INTERNAL = (_TAGS_KEY, _PARENT_KEY, "_wandb")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# Runs of this process. Shared by every WandbBackend instance, because authtracker may create a new
|
|
34
|
+
# backend object per call (for example for best_run) and it must still see runs started earlier.
|
|
35
|
+
_PROCESS_RECORDS: Dict[str, Dict[str, Any]] = {}
|
|
36
|
+
_PROCESS_RUNS: Dict[str, Any] = {}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _reset_process_state() -> None:
|
|
40
|
+
"""Forget this process's runs (used by tests)."""
|
|
41
|
+
_PROCESS_RECORDS.clear()
|
|
42
|
+
_PROCESS_RUNS.clear()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _wb_tags(tags: Dict[str, str]) -> List[str]:
|
|
46
|
+
"""W&B tags are plain strings (max 64 chars), so key/value pairs become 'key=value'."""
|
|
47
|
+
return [f"{k}={v}"[:64] for k, v in tags.items()]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class WandbBackend(Backend):
|
|
51
|
+
buffers_offline = True # W&B has its own offline mode, so no extra authtracker queue
|
|
52
|
+
captures_console = True # W&B records print/logging output itself (the run's Logs tab)
|
|
53
|
+
|
|
54
|
+
def __init__(self, settings: Settings, api: Any = None):
|
|
55
|
+
import wandb # noqa: F401 (ImportError is turned into a helpful message by the factory)
|
|
56
|
+
self._wandb = wandb
|
|
57
|
+
self._settings = settings
|
|
58
|
+
self._api_obj = api
|
|
59
|
+
self._records = _PROCESS_RECORDS # latest values of this process's runs
|
|
60
|
+
self._runs = _PROCESS_RUNS # run id -> open wandb run
|
|
61
|
+
|
|
62
|
+
# ------------------------------------------------------------------ helpers
|
|
63
|
+
@property
|
|
64
|
+
def _offline(self) -> bool:
|
|
65
|
+
return os.environ.get("WANDB_MODE", "").lower() in ("offline", "disabled", "dryrun")
|
|
66
|
+
|
|
67
|
+
def _api(self):
|
|
68
|
+
if self._api_obj is None:
|
|
69
|
+
overrides = {"base_url": self._settings.tracking_uri} if self._settings.tracking_uri else {}
|
|
70
|
+
self._api_obj = self._wandb.Api(overrides=overrides)
|
|
71
|
+
return self._api_obj
|
|
72
|
+
|
|
73
|
+
def _entity(self) -> str:
|
|
74
|
+
return os.environ.get("WANDB_ENTITY") or self._api().default_entity
|
|
75
|
+
|
|
76
|
+
@staticmethod
|
|
77
|
+
def _split(run_id: str):
|
|
78
|
+
project, _, wb_id = run_id.partition("/")
|
|
79
|
+
return project, wb_id
|
|
80
|
+
|
|
81
|
+
def _open(self, run_id: str):
|
|
82
|
+
try:
|
|
83
|
+
return self._runs[run_id]
|
|
84
|
+
except KeyError:
|
|
85
|
+
raise BackendError(f"Run {run_id} is not open in this process") from None
|
|
86
|
+
|
|
87
|
+
@staticmethod
|
|
88
|
+
def _public(rec: Dict[str, Any]) -> Dict[str, Any]:
|
|
89
|
+
return {"run_id": rec["run_id"], "name": rec["name"], "metrics": dict(rec["metrics"]),
|
|
90
|
+
"params": dict(rec["params"]), "tags": dict(rec["tags"])}
|
|
91
|
+
|
|
92
|
+
@staticmethod
|
|
93
|
+
def _from_remote(r: Any, project: str) -> Dict[str, Any]:
|
|
94
|
+
config = dict(r.config)
|
|
95
|
+
summary = dict(r.summary)
|
|
96
|
+
return {
|
|
97
|
+
"run_id": f"{project}/{r.id}",
|
|
98
|
+
"name": r.name,
|
|
99
|
+
"tags": {k: str(v) for k, v in (config.get(_TAGS_KEY) or {}).items()},
|
|
100
|
+
"params": {k: str(v) for k, v in config.items() if k not in _INTERNAL},
|
|
101
|
+
"metrics": {k: float(v) for k, v in summary.items()
|
|
102
|
+
if isinstance(v, (int, float)) and not isinstance(v, bool)
|
|
103
|
+
and not k.startswith("_")},
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
# ------------------------------------------------------------------ required
|
|
107
|
+
def ping(self) -> None:
|
|
108
|
+
if self._offline:
|
|
109
|
+
return
|
|
110
|
+
self._api().viewer # noqa: B018 (raises when the key or server is wrong)
|
|
111
|
+
|
|
112
|
+
def start_run(self, experiment, run_name, tags, parent_id):
|
|
113
|
+
config: Dict[str, Any] = {_TAGS_KEY: dict(tags)}
|
|
114
|
+
if parent_id:
|
|
115
|
+
config[_PARENT_KEY] = parent_id
|
|
116
|
+
wb_settings = self._wandb.Settings(
|
|
117
|
+
silent=True, **({"base_url": self._settings.tracking_uri}
|
|
118
|
+
if self._settings.tracking_uri else {}))
|
|
119
|
+
# The SDK reads WANDB_DIR / WANDB_MODE once per process; pass them on every call instead.
|
|
120
|
+
env_args = {k: os.environ[v] for k, v in (("dir", "WANDB_DIR"), ("mode", "WANDB_MODE"))
|
|
121
|
+
if os.environ.get(v)}
|
|
122
|
+
try:
|
|
123
|
+
run = self._wandb.init(
|
|
124
|
+
project=experiment, name=run_name, group=tags.get("experiment"),
|
|
125
|
+
job_type=tags.get(TAG_LEVEL), tags=_wb_tags(tags), config=config,
|
|
126
|
+
reinit="create_new", settings=wb_settings, **env_args)
|
|
127
|
+
except TypeError as e:
|
|
128
|
+
raise BackendError("This backend needs a newer wandb (several runs open at once): "
|
|
129
|
+
"pip install -U wandb") from e
|
|
130
|
+
run_id = f"{experiment}/{run.id}"
|
|
131
|
+
self._runs[run_id] = run
|
|
132
|
+
self._records[run_id] = {"run_id": run_id, "name": run_name, "project": experiment,
|
|
133
|
+
"tags": {k: str(v) for k, v in tags.items()},
|
|
134
|
+
"params": {}, "metrics": {}}
|
|
135
|
+
return run_id
|
|
136
|
+
|
|
137
|
+
def end_run(self, run_id, status):
|
|
138
|
+
run = self._runs.pop(run_id, None)
|
|
139
|
+
if run is not None:
|
|
140
|
+
run.finish(exit_code=0 if status == "FINISHED" else 1)
|
|
141
|
+
|
|
142
|
+
def set_tags(self, run_id, tags):
|
|
143
|
+
run, rec = self._open(run_id), self._records[run_id]
|
|
144
|
+
rec["tags"].update({k: str(v) for k, v in tags.items()})
|
|
145
|
+
run.config.update({_TAGS_KEY: dict(rec["tags"])}, allow_val_change=True)
|
|
146
|
+
run.tags = tuple(_wb_tags(rec["tags"]))
|
|
147
|
+
|
|
148
|
+
def log_params(self, run_id, params):
|
|
149
|
+
run, rec = self._open(run_id), self._records[run_id]
|
|
150
|
+
for k, v in params.items():
|
|
151
|
+
run.config.update({k: v}) # W&B raises if a value is changed
|
|
152
|
+
rec["params"][k] = str(v)
|
|
153
|
+
|
|
154
|
+
def log_metrics(self, run_id, metrics, step):
|
|
155
|
+
run, rec = self._open(run_id), self._records[run_id]
|
|
156
|
+
values = {k: float(v) for k, v in metrics.items()}
|
|
157
|
+
run.log(values) if step is None else run.log(values, step=step)
|
|
158
|
+
rec["metrics"].update(values)
|
|
159
|
+
|
|
160
|
+
def log_artifact(self, run_id, path):
|
|
161
|
+
run = self._open(run_id)
|
|
162
|
+
dest = os.path.join(run.dir, os.path.basename(path)) # copy: the caller may delete it
|
|
163
|
+
shutil.copy(path, dest)
|
|
164
|
+
run.save(dest, base_path=run.dir, policy="now")
|
|
165
|
+
|
|
166
|
+
def get_run(self, run_id):
|
|
167
|
+
if run_id in self._records:
|
|
168
|
+
return self._public(self._records[run_id])
|
|
169
|
+
project, wb_id = self._split(run_id)
|
|
170
|
+
return self._from_remote(self._api().run(f"{self._entity()}/{project}/{wb_id}"), project)
|
|
171
|
+
|
|
172
|
+
def search_runs(self, experiment, tags, metric, mode, limit):
|
|
173
|
+
found: Dict[str, Dict[str, Any]] = {}
|
|
174
|
+
if not self._offline:
|
|
175
|
+
try:
|
|
176
|
+
remote = list(self._api().runs(f"{self._entity()}/{experiment}"))
|
|
177
|
+
except Exception as e:
|
|
178
|
+
if "not find" not in str(e).lower() and "not found" not in str(e).lower():
|
|
179
|
+
raise
|
|
180
|
+
remote = []
|
|
181
|
+
for r in remote:
|
|
182
|
+
s = self._from_remote(r, experiment)
|
|
183
|
+
found[s["run_id"]] = s
|
|
184
|
+
for rec in self._records.values(): # this process's runs win over the server's copy
|
|
185
|
+
if rec["project"] == experiment:
|
|
186
|
+
found[rec["run_id"]] = self._public(rec)
|
|
187
|
+
hits = [s for s in found.values() if metric in s["metrics"]
|
|
188
|
+
and all(s["tags"].get(k) == v for k, v in tags.items())]
|
|
189
|
+
hits.sort(key=lambda s: s["metrics"][metric], reverse=(mode == "max"))
|
|
190
|
+
return hits[:limit]
|
|
191
|
+
|
|
192
|
+
# ------------------------------------------------------------------ optional: models
|
|
193
|
+
def log_model(self, run_id, model, flavor, **kwargs):
|
|
194
|
+
"""Save a model as a W&B artifact (type 'model').
|
|
195
|
+
|
|
196
|
+
`model` is a file or folder path, a torch module (saved with torch.save) or any picklable
|
|
197
|
+
object. Options: code_paths=[files or folders] (stored under code/) and
|
|
198
|
+
pip_requirements=[...]. W&B has no model flavors, so `flavor` is ignored.
|
|
199
|
+
"""
|
|
200
|
+
run, rec = self._open(run_id), self._records[run_id]
|
|
201
|
+
code_paths = kwargs.pop("code_paths", None) or []
|
|
202
|
+
requirements = kwargs.pop("pip_requirements", None)
|
|
203
|
+
if kwargs:
|
|
204
|
+
raise BackendError(f"The wandb backend does not support log_model options {sorted(kwargs)}"
|
|
205
|
+
" (supported: code_paths, pip_requirements)")
|
|
206
|
+
name = f"model-{run_id.split('/', 1)[1]}"
|
|
207
|
+
art = self._wandb.Artifact(name, type="model", metadata={"run": run_id, **{
|
|
208
|
+
k: v for k, v in rec["tags"].items() if k in ("model", "experiment")}})
|
|
209
|
+
if isinstance(model, (str, os.PathLike)) and os.path.exists(model):
|
|
210
|
+
path = os.fspath(model)
|
|
211
|
+
art.add_dir(path, name="model") if os.path.isdir(path) else art.add_file(path, name=os.path.basename(path))
|
|
212
|
+
elif type(model).__module__.split(".")[0] == "torch" and hasattr(model, "state_dict"):
|
|
213
|
+
import torch
|
|
214
|
+
with art.new_file("model.pt", "wb") as f:
|
|
215
|
+
torch.save(model, f)
|
|
216
|
+
else:
|
|
217
|
+
try:
|
|
218
|
+
import cloudpickle as pickler
|
|
219
|
+
except ImportError:
|
|
220
|
+
pickler = pickle
|
|
221
|
+
with art.new_file("model.pkl", "wb") as f:
|
|
222
|
+
pickler.dump(model, f)
|
|
223
|
+
for p in code_paths:
|
|
224
|
+
base = os.path.basename(os.fspath(p).rstrip("/"))
|
|
225
|
+
if os.path.isdir(p):
|
|
226
|
+
art.add_dir(os.fspath(p), name=f"code/{base}")
|
|
227
|
+
else:
|
|
228
|
+
art.add_file(os.fspath(p), name=f"code/{base}")
|
|
229
|
+
if requirements:
|
|
230
|
+
with art.new_file("requirements.txt") as f:
|
|
231
|
+
f.write("\n".join(requirements) + "\n")
|
|
232
|
+
run.log_artifact(art)
|
|
233
|
+
uri = f"wandb-artifact:{run_id.split('/', 1)[0]}/{name}"
|
|
234
|
+
self.set_tags(run_id, {TAG_MODEL_URI: uri})
|
|
235
|
+
return uri
|