authtracker 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,235 @@
1
+ """Weights & Biases backend. Requires: pip install wandb
2
+
3
+ Mapping (same three levels as every backend):
4
+
5
+ model -> W&B project
6
+ experiment -> run group; the parent run (job type "experiment")
7
+ run -> a run (job type "run") in that group
8
+
9
+ Settings: sign in with WANDB_API_KEY, optional WANDB_ENTITY (team/user), WANDB_MODE=offline to
10
+ write locally and `wandb sync` later. AUTHTRACKER_TRACKING_URI is used as the W&B base URL for a
11
+ self-hosted server. Run ids look like "<project>/<wandb run id>".
12
+
13
+ Supported: runs, params, metrics, tags, files, best_run, and models saved as artifacts.
14
+ Not supported yet: the model registry (register / promote / rollback / load). Those raise a clear
15
+ BackendError.
16
+ """
17
+ from __future__ import annotations
18
+
19
+ import os
20
+ import pickle
21
+ import shutil
22
+ from typing import Any, Dict, List, Optional
23
+
24
+ from ..errors import BackendError
25
+ from ..settings import Settings
26
+ from .base import TAG_LEVEL, TAG_MODEL_URI, Backend
27
+
28
+ _TAGS_KEY = "authtracker_tags" # our key/value tags, stored in the run config
29
+ _PARENT_KEY = "authtracker_parent"
30
+ _INTERNAL = (_TAGS_KEY, _PARENT_KEY, "_wandb")
31
+
32
+
33
+ # Runs of this process. Shared by every WandbBackend instance, because authtracker may create a new
34
+ # backend object per call (for example for best_run) and it must still see runs started earlier.
35
+ _PROCESS_RECORDS: Dict[str, Dict[str, Any]] = {}
36
+ _PROCESS_RUNS: Dict[str, Any] = {}
37
+
38
+
39
+ def _reset_process_state() -> None:
40
+ """Forget this process's runs (used by tests)."""
41
+ _PROCESS_RECORDS.clear()
42
+ _PROCESS_RUNS.clear()
43
+
44
+
45
+ def _wb_tags(tags: Dict[str, str]) -> List[str]:
46
+ """W&B tags are plain strings (max 64 chars), so key/value pairs become 'key=value'."""
47
+ return [f"{k}={v}"[:64] for k, v in tags.items()]
48
+
49
+
50
+ class WandbBackend(Backend):
51
+ buffers_offline = True # W&B has its own offline mode, so no extra authtracker queue
52
+ captures_console = True # W&B records print/logging output itself (the run's Logs tab)
53
+
54
+ def __init__(self, settings: Settings, api: Any = None):
55
+ import wandb # noqa: F401 (ImportError is turned into a helpful message by the factory)
56
+ self._wandb = wandb
57
+ self._settings = settings
58
+ self._api_obj = api
59
+ self._records = _PROCESS_RECORDS # latest values of this process's runs
60
+ self._runs = _PROCESS_RUNS # run id -> open wandb run
61
+
62
+ # ------------------------------------------------------------------ helpers
63
+ @property
64
+ def _offline(self) -> bool:
65
+ return os.environ.get("WANDB_MODE", "").lower() in ("offline", "disabled", "dryrun")
66
+
67
+ def _api(self):
68
+ if self._api_obj is None:
69
+ overrides = {"base_url": self._settings.tracking_uri} if self._settings.tracking_uri else {}
70
+ self._api_obj = self._wandb.Api(overrides=overrides)
71
+ return self._api_obj
72
+
73
+ def _entity(self) -> str:
74
+ return os.environ.get("WANDB_ENTITY") or self._api().default_entity
75
+
76
+ @staticmethod
77
+ def _split(run_id: str):
78
+ project, _, wb_id = run_id.partition("/")
79
+ return project, wb_id
80
+
81
+ def _open(self, run_id: str):
82
+ try:
83
+ return self._runs[run_id]
84
+ except KeyError:
85
+ raise BackendError(f"Run {run_id} is not open in this process") from None
86
+
87
+ @staticmethod
88
+ def _public(rec: Dict[str, Any]) -> Dict[str, Any]:
89
+ return {"run_id": rec["run_id"], "name": rec["name"], "metrics": dict(rec["metrics"]),
90
+ "params": dict(rec["params"]), "tags": dict(rec["tags"])}
91
+
92
+ @staticmethod
93
+ def _from_remote(r: Any, project: str) -> Dict[str, Any]:
94
+ config = dict(r.config)
95
+ summary = dict(r.summary)
96
+ return {
97
+ "run_id": f"{project}/{r.id}",
98
+ "name": r.name,
99
+ "tags": {k: str(v) for k, v in (config.get(_TAGS_KEY) or {}).items()},
100
+ "params": {k: str(v) for k, v in config.items() if k not in _INTERNAL},
101
+ "metrics": {k: float(v) for k, v in summary.items()
102
+ if isinstance(v, (int, float)) and not isinstance(v, bool)
103
+ and not k.startswith("_")},
104
+ }
105
+
106
+ # ------------------------------------------------------------------ required
107
+ def ping(self) -> None:
108
+ if self._offline:
109
+ return
110
+ self._api().viewer # noqa: B018 (raises when the key or server is wrong)
111
+
112
+ def start_run(self, experiment, run_name, tags, parent_id):
113
+ config: Dict[str, Any] = {_TAGS_KEY: dict(tags)}
114
+ if parent_id:
115
+ config[_PARENT_KEY] = parent_id
116
+ wb_settings = self._wandb.Settings(
117
+ silent=True, **({"base_url": self._settings.tracking_uri}
118
+ if self._settings.tracking_uri else {}))
119
+ # The SDK reads WANDB_DIR / WANDB_MODE once per process; pass them on every call instead.
120
+ env_args = {k: os.environ[v] for k, v in (("dir", "WANDB_DIR"), ("mode", "WANDB_MODE"))
121
+ if os.environ.get(v)}
122
+ try:
123
+ run = self._wandb.init(
124
+ project=experiment, name=run_name, group=tags.get("experiment"),
125
+ job_type=tags.get(TAG_LEVEL), tags=_wb_tags(tags), config=config,
126
+ reinit="create_new", settings=wb_settings, **env_args)
127
+ except TypeError as e:
128
+ raise BackendError("This backend needs a newer wandb (several runs open at once): "
129
+ "pip install -U wandb") from e
130
+ run_id = f"{experiment}/{run.id}"
131
+ self._runs[run_id] = run
132
+ self._records[run_id] = {"run_id": run_id, "name": run_name, "project": experiment,
133
+ "tags": {k: str(v) for k, v in tags.items()},
134
+ "params": {}, "metrics": {}}
135
+ return run_id
136
+
137
+ def end_run(self, run_id, status):
138
+ run = self._runs.pop(run_id, None)
139
+ if run is not None:
140
+ run.finish(exit_code=0 if status == "FINISHED" else 1)
141
+
142
+ def set_tags(self, run_id, tags):
143
+ run, rec = self._open(run_id), self._records[run_id]
144
+ rec["tags"].update({k: str(v) for k, v in tags.items()})
145
+ run.config.update({_TAGS_KEY: dict(rec["tags"])}, allow_val_change=True)
146
+ run.tags = tuple(_wb_tags(rec["tags"]))
147
+
148
+ def log_params(self, run_id, params):
149
+ run, rec = self._open(run_id), self._records[run_id]
150
+ for k, v in params.items():
151
+ run.config.update({k: v}) # W&B raises if a value is changed
152
+ rec["params"][k] = str(v)
153
+
154
+ def log_metrics(self, run_id, metrics, step):
155
+ run, rec = self._open(run_id), self._records[run_id]
156
+ values = {k: float(v) for k, v in metrics.items()}
157
+ run.log(values) if step is None else run.log(values, step=step)
158
+ rec["metrics"].update(values)
159
+
160
+ def log_artifact(self, run_id, path):
161
+ run = self._open(run_id)
162
+ dest = os.path.join(run.dir, os.path.basename(path)) # copy: the caller may delete it
163
+ shutil.copy(path, dest)
164
+ run.save(dest, base_path=run.dir, policy="now")
165
+
166
+ def get_run(self, run_id):
167
+ if run_id in self._records:
168
+ return self._public(self._records[run_id])
169
+ project, wb_id = self._split(run_id)
170
+ return self._from_remote(self._api().run(f"{self._entity()}/{project}/{wb_id}"), project)
171
+
172
+ def search_runs(self, experiment, tags, metric, mode, limit):
173
+ found: Dict[str, Dict[str, Any]] = {}
174
+ if not self._offline:
175
+ try:
176
+ remote = list(self._api().runs(f"{self._entity()}/{experiment}"))
177
+ except Exception as e:
178
+ if "not find" not in str(e).lower() and "not found" not in str(e).lower():
179
+ raise
180
+ remote = []
181
+ for r in remote:
182
+ s = self._from_remote(r, experiment)
183
+ found[s["run_id"]] = s
184
+ for rec in self._records.values(): # this process's runs win over the server's copy
185
+ if rec["project"] == experiment:
186
+ found[rec["run_id"]] = self._public(rec)
187
+ hits = [s for s in found.values() if metric in s["metrics"]
188
+ and all(s["tags"].get(k) == v for k, v in tags.items())]
189
+ hits.sort(key=lambda s: s["metrics"][metric], reverse=(mode == "max"))
190
+ return hits[:limit]
191
+
192
+ # ------------------------------------------------------------------ optional: models
193
+ def log_model(self, run_id, model, flavor, **kwargs):
194
+ """Save a model as a W&B artifact (type 'model').
195
+
196
+ `model` is a file or folder path, a torch module (saved with torch.save) or any picklable
197
+ object. Options: code_paths=[files or folders] (stored under code/) and
198
+ pip_requirements=[...]. W&B has no model flavors, so `flavor` is ignored.
199
+ """
200
+ run, rec = self._open(run_id), self._records[run_id]
201
+ code_paths = kwargs.pop("code_paths", None) or []
202
+ requirements = kwargs.pop("pip_requirements", None)
203
+ if kwargs:
204
+ raise BackendError(f"The wandb backend does not support log_model options {sorted(kwargs)}"
205
+ " (supported: code_paths, pip_requirements)")
206
+ name = f"model-{run_id.split('/', 1)[1]}"
207
+ art = self._wandb.Artifact(name, type="model", metadata={"run": run_id, **{
208
+ k: v for k, v in rec["tags"].items() if k in ("model", "experiment")}})
209
+ if isinstance(model, (str, os.PathLike)) and os.path.exists(model):
210
+ path = os.fspath(model)
211
+ art.add_dir(path, name="model") if os.path.isdir(path) else art.add_file(path, name=os.path.basename(path))
212
+ elif type(model).__module__.split(".")[0] == "torch" and hasattr(model, "state_dict"):
213
+ import torch
214
+ with art.new_file("model.pt", "wb") as f:
215
+ torch.save(model, f)
216
+ else:
217
+ try:
218
+ import cloudpickle as pickler
219
+ except ImportError:
220
+ pickler = pickle
221
+ with art.new_file("model.pkl", "wb") as f:
222
+ pickler.dump(model, f)
223
+ for p in code_paths:
224
+ base = os.path.basename(os.fspath(p).rstrip("/"))
225
+ if os.path.isdir(p):
226
+ art.add_dir(os.fspath(p), name=f"code/{base}")
227
+ else:
228
+ art.add_file(os.fspath(p), name=f"code/{base}")
229
+ if requirements:
230
+ with art.new_file("requirements.txt") as f:
231
+ f.write("\n".join(requirements) + "\n")
232
+ run.log_artifact(art)
233
+ uri = f"wandb-artifact:{run_id.split('/', 1)[0]}/{name}"
234
+ self.set_tags(run_id, {TAG_MODEL_URI: uri})
235
+ return uri