authtracker 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- authtracker/__init__.py +16 -0
- authtracker/backends/__init__.py +95 -0
- authtracker/backends/base.py +136 -0
- authtracker/backends/local.py +200 -0
- authtracker/backends/mlflow_backend.py +310 -0
- authtracker/backends/resilient.py +329 -0
- authtracker/backends/wandb_backend.py +235 -0
- authtracker/bundle.py +887 -0
- authtracker/capture.py +175 -0
- authtracker/cli.py +51 -0
- authtracker/configfile.py +84 -0
- authtracker/dataset.py +98 -0
- authtracker/errors.py +18 -0
- authtracker/registry.py +118 -0
- authtracker/run.py +346 -0
- authtracker/settings.py +171 -0
- authtracker/simple.py +241 -0
- authtracker/testing.py +82 -0
- authtracker/utils.py +321 -0
- authtracker/watch.py +59 -0
- authtracker-0.0.1.dist-info/METADATA +215 -0
- authtracker-0.0.1.dist-info/RECORD +25 -0
- authtracker-0.0.1.dist-info/WHEEL +4 -0
- authtracker-0.0.1.dist-info/entry_points.txt +2 -0
- authtracker-0.0.1.dist-info/licenses/LICENSE +202 -0
authtracker/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""authtracker: variable-configured experiment tracking and model versioning."""
|
|
2
|
+
from .errors import AuthTrackerError, BackendError, PromotionError, UnsupportedError, ValidationError
|
|
3
|
+
from .bundle import load_dir as load_bundle
|
|
4
|
+
from .registry import best_run, load, promote, register, rollback
|
|
5
|
+
from .run import Run, run
|
|
6
|
+
from .simple import current_run, finish, init, log, restore, save, set_tags, start, watch
|
|
7
|
+
from .settings import ConfigError, Settings, configure, get_settings
|
|
8
|
+
|
|
9
|
+
__version__ = "0.0.1"
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"init", "start", "log", "save", "restore", "watch", "finish", "set_tags", "current_run",
|
|
13
|
+
"run", "Run", "register", "promote", "rollback", "load", "load_bundle", "best_run",
|
|
14
|
+
"Settings", "configure", "get_settings", "ConfigError",
|
|
15
|
+
"AuthTrackerError", "ValidationError", "PromotionError", "BackendError", "UnsupportedError",
|
|
16
|
+
]
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Backend lookup: built-ins, runtime registration, and installed plugins.
|
|
2
|
+
|
|
3
|
+
Resolution order for AUTHTRACKER_BACKEND=<name>:
|
|
4
|
+
1. a backend registered in code with `register_backend`;
|
|
5
|
+
2. an installed package that declares it in the `authtracker.backends` entry-point group, e.g.
|
|
6
|
+
in that package's pyproject.toml:
|
|
7
|
+
|
|
8
|
+
[project.entry-points."authtracker.backends"]
|
|
9
|
+
wandb = "authtracker_wandb:make_backend" # callable(settings) -> Backend
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from importlib import metadata
|
|
14
|
+
from typing import Callable, Dict, List, Optional
|
|
15
|
+
|
|
16
|
+
from ..errors import BackendError
|
|
17
|
+
from ..settings import Settings
|
|
18
|
+
from .base import BACKEND_API_VERSION, Backend
|
|
19
|
+
|
|
20
|
+
ENTRY_POINT_GROUP = "authtracker.backends"
|
|
21
|
+
|
|
22
|
+
_REGISTRY: Dict[str, Callable[[Settings], Backend]] = {}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def register_backend(name: str, factory: Callable[[Settings], Backend]) -> None:
|
|
26
|
+
_REGISTRY[name] = factory
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _make_local(settings: Settings) -> Backend:
|
|
30
|
+
from .local import LocalBackend
|
|
31
|
+
return LocalBackend(settings)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _make_mlflow(settings: Settings) -> Backend:
|
|
35
|
+
try:
|
|
36
|
+
from .mlflow_backend import MlflowBackend
|
|
37
|
+
except ImportError as e: # mlflow not installed
|
|
38
|
+
raise BackendError(
|
|
39
|
+
"Backend 'mlflow' needs MLflow. Install one of: pip install 'authtracker[mlflow]' "
|
|
40
|
+
"(full), or pip install 'authtracker[mlflow-skinny]' (light: only sends runs to a server)"
|
|
41
|
+
) from e
|
|
42
|
+
return MlflowBackend(settings)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _make_wandb(settings: Settings) -> Backend:
|
|
46
|
+
try:
|
|
47
|
+
from .wandb_backend import WandbBackend
|
|
48
|
+
except ImportError as e: # wandb not installed
|
|
49
|
+
raise BackendError("Backend 'wandb' needs W&B: pip install 'authtracker[wandb]'") from e
|
|
50
|
+
return WandbBackend(settings)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
register_backend("mlflow", _make_mlflow)
|
|
54
|
+
register_backend("local", _make_local)
|
|
55
|
+
register_backend("wandb", _make_wandb)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _entry_points() -> List[metadata.EntryPoint]:
|
|
59
|
+
try:
|
|
60
|
+
return list(metadata.entry_points(group=ENTRY_POINT_GROUP))
|
|
61
|
+
except TypeError: # Python < 3.10
|
|
62
|
+
return list(metadata.entry_points().get(ENTRY_POINT_GROUP, []))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def available_backends() -> List[str]:
|
|
66
|
+
return sorted(set(_REGISTRY) | {ep.name for ep in _entry_points()})
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _find_factory(name: str) -> Optional[Callable[[Settings], Backend]]:
|
|
70
|
+
if name in _REGISTRY:
|
|
71
|
+
return _REGISTRY[name]
|
|
72
|
+
for ep in _entry_points():
|
|
73
|
+
if ep.name == name:
|
|
74
|
+
return ep.load()
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def get_backend(settings: Settings) -> Backend:
|
|
79
|
+
factory = _find_factory(settings.backend)
|
|
80
|
+
if factory is None:
|
|
81
|
+
raise BackendError(
|
|
82
|
+
f"Unknown backend {settings.backend!r}. Available: {available_backends()}")
|
|
83
|
+
backend = factory(settings)
|
|
84
|
+
version = getattr(backend, "api_version", None)
|
|
85
|
+
if version != BACKEND_API_VERSION:
|
|
86
|
+
raise BackendError(
|
|
87
|
+
f"Backend {settings.backend!r} implements backend API version {version}, "
|
|
88
|
+
f"but this authtracker needs version {BACKEND_API_VERSION}")
|
|
89
|
+
if settings.offline_queue and not backend.buffers_offline:
|
|
90
|
+
from .resilient import ResilientBackend
|
|
91
|
+
backend = ResilientBackend(backend, settings)
|
|
92
|
+
return backend
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
__all__ = ["Backend", "get_backend", "register_backend", "available_backends"]
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""The only interface the rest of authtracker talks to.
|
|
2
|
+
|
|
3
|
+
To support another tracking platform, subclass `Backend`:
|
|
4
|
+
|
|
5
|
+
* implement the REQUIRED methods (runs, params, metrics, tags, artifacts, queries);
|
|
6
|
+
* optionally implement the model methods (`log_model`) and the registry methods
|
|
7
|
+
(`register`, `set_alias`, `get_alias`, `version_run_id`, `load`). Anything you leave out
|
|
8
|
+
raises a clear BackendError when a user calls it;
|
|
9
|
+
* register it with `register_backend("name", factory)`, or ship it in a separate package via the
|
|
10
|
+
`authtracker.backends` entry-point group (no change to authtracker needed);
|
|
11
|
+
* check it with `authtracker.testing.check_backend(your_backend)`.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from abc import ABC, abstractmethod
|
|
16
|
+
from typing import Any, Dict, List, Optional
|
|
17
|
+
|
|
18
|
+
from ..errors import BackendError, UnsupportedError
|
|
19
|
+
|
|
20
|
+
# Bump when a change to this interface would break existing backends.
|
|
21
|
+
BACKEND_API_VERSION = 1
|
|
22
|
+
|
|
23
|
+
# Tags the package itself writes.
|
|
24
|
+
TAG_LEVEL = "authtracker.level" # "experiment" (parent) or "run" (child)
|
|
25
|
+
TAG_MODEL_URI = "authtracker.model_uri" # where log_model put the model
|
|
26
|
+
TAG_RUN_ID = "authtracker.run_id" # on model versions: source run
|
|
27
|
+
|
|
28
|
+
_TRANSIENT_NAMES = {"ConnectionError", "Timeout", "ConnectTimeout", "ReadTimeout",
|
|
29
|
+
"TimeoutError", "OperationalError"}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Backend(ABC):
|
|
33
|
+
api_version = BACKEND_API_VERSION
|
|
34
|
+
# True if the platform already buffers writes while offline (for example W&B's offline mode).
|
|
35
|
+
# authtracker then skips its own retry/queue layer for this backend.
|
|
36
|
+
buffers_offline = False
|
|
37
|
+
# True if the platform already records everything the program prints (W&B does). authtracker
|
|
38
|
+
# then does not copy the console output into an `output.log` artifact itself.
|
|
39
|
+
captures_console = False
|
|
40
|
+
|
|
41
|
+
def is_transient(self, exc: Exception) -> bool:
|
|
42
|
+
"""True if `exc` looks like a temporary outage (worth retrying / queuing)."""
|
|
43
|
+
return any(c.__name__ in _TRANSIENT_NAMES for c in type(exc).__mro__)
|
|
44
|
+
|
|
45
|
+
def _unsupported(self, what: str) -> BackendError:
|
|
46
|
+
return UnsupportedError(f"The {type(self).__name__} backend does not support {what}")
|
|
47
|
+
|
|
48
|
+
# ------------------------------------------------------------------ required
|
|
49
|
+
@abstractmethod
|
|
50
|
+
def ping(self) -> None:
|
|
51
|
+
"""Raise if the backend is unreachable."""
|
|
52
|
+
|
|
53
|
+
@abstractmethod
|
|
54
|
+
def start_run(self, experiment: str, run_name: str, tags: Dict[str, str],
|
|
55
|
+
parent_id: Optional[str]) -> str:
|
|
56
|
+
"""Create a run in `experiment` (the model name) and return its id."""
|
|
57
|
+
|
|
58
|
+
@abstractmethod
|
|
59
|
+
def end_run(self, run_id: str, status: str) -> None:
|
|
60
|
+
"""Status is FINISHED, FAILED or KILLED."""
|
|
61
|
+
|
|
62
|
+
@abstractmethod
|
|
63
|
+
def set_tags(self, run_id: str, tags: Dict[str, str]) -> None: ...
|
|
64
|
+
|
|
65
|
+
@abstractmethod
|
|
66
|
+
def log_params(self, run_id: str, params: Dict[str, Any]) -> None: ...
|
|
67
|
+
|
|
68
|
+
@abstractmethod
|
|
69
|
+
def log_metrics(self, run_id: str, metrics: Dict[str, float], step: Optional[int]) -> None: ...
|
|
70
|
+
|
|
71
|
+
@abstractmethod
|
|
72
|
+
def log_artifact(self, run_id: str, path: str) -> None:
|
|
73
|
+
"""Store the file at `path` under its own file name."""
|
|
74
|
+
|
|
75
|
+
@abstractmethod
|
|
76
|
+
def get_run(self, run_id: str) -> Dict[str, Any]:
|
|
77
|
+
"""Return {'run_id','name','metrics','params','tags'} (metrics = latest values)."""
|
|
78
|
+
|
|
79
|
+
@abstractmethod
|
|
80
|
+
def search_runs(self, experiment: str, tags: Dict[str, str], metric: str,
|
|
81
|
+
mode: str, limit: int) -> List[Dict[str, Any]]:
|
|
82
|
+
"""Runs in `experiment` having all `tags` and a value for `metric`, best first
|
|
83
|
+
('max' = highest first, 'min' = lowest first), at most `limit`, same shape as get_run.
|
|
84
|
+
An unknown experiment gives []."""
|
|
85
|
+
|
|
86
|
+
# ------------------------------------------------------------------ optional: resuming runs
|
|
87
|
+
def find_run(self, experiment: str, name: str, tags: Dict[str, str],
|
|
88
|
+
parent_id: Optional[str]) -> Optional[str]:
|
|
89
|
+
"""Id of the newest run in `experiment` that is called `name`, has all `tags` and is a child of
|
|
90
|
+
`parent_id` (None for a top-level run); None if there is no such run."""
|
|
91
|
+
raise self._unsupported("resuming runs")
|
|
92
|
+
|
|
93
|
+
def resume_run(self, run_id: str, parent_id: Optional[str]) -> str:
|
|
94
|
+
"""Reopen a finished run so that more can be logged to it, and return its id. If `parent_id` is
|
|
95
|
+
given, that run is the open run it is nested in."""
|
|
96
|
+
raise self._unsupported("resuming runs")
|
|
97
|
+
|
|
98
|
+
def last_step(self, run_id: str) -> Optional[int]:
|
|
99
|
+
"""The highest metric step logged to the run so far (None: no metrics, or not known)."""
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
def download_artifact(self, run_id: str, name: str, folder: str) -> Optional[str]:
|
|
103
|
+
"""Copy the file `name` that was saved on the run into `folder` and return its path; None if the run has no such file."""
|
|
104
|
+
raise self._unsupported("downloading files")
|
|
105
|
+
|
|
106
|
+
# ------------------------------------------------------------------ optional: models
|
|
107
|
+
def log_model(self, run_id: str, model: Any, flavor: Optional[str], **kwargs) -> str:
|
|
108
|
+
"""Log a model for the run, set the TAG_MODEL_URI tag on it and return the URI."""
|
|
109
|
+
raise self._unsupported("logging models")
|
|
110
|
+
|
|
111
|
+
def log_model_dir(self, run_id: str, local_dir: str) -> str:
|
|
112
|
+
"""Store a folder as the run's model (a PyTorch bundle), set TAG_MODEL_URI and return its URI.
|
|
113
|
+
`register` must then be able to make a version from that URI."""
|
|
114
|
+
raise self._unsupported("saving model bundles")
|
|
115
|
+
|
|
116
|
+
def download_bundle(self, name: str, alias: str, dst: Optional[str] = None) -> Optional[str]:
|
|
117
|
+
"""If version `alias` of `name` is a model bundle, copy it into a NEW folder and return its
|
|
118
|
+
path (the caller removes it); otherwise return None."""
|
|
119
|
+
return None
|
|
120
|
+
|
|
121
|
+
# ------------------------------------------------------------------ optional: registry
|
|
122
|
+
def register(self, run_id: str, name: str, model_uri: str) -> str:
|
|
123
|
+
"""Create a new model version and return its version string."""
|
|
124
|
+
raise self._unsupported("the model registry")
|
|
125
|
+
|
|
126
|
+
def set_alias(self, name: str, version: str, alias: str) -> None:
|
|
127
|
+
raise self._unsupported("the model registry")
|
|
128
|
+
|
|
129
|
+
def get_alias(self, name: str, alias: str) -> Optional[str]:
|
|
130
|
+
raise self._unsupported("the model registry")
|
|
131
|
+
|
|
132
|
+
def version_run_id(self, name: str, version: str) -> str:
|
|
133
|
+
raise self._unsupported("the model registry")
|
|
134
|
+
|
|
135
|
+
def load(self, name: str, alias: str) -> Any:
|
|
136
|
+
raise self._unsupported("the model registry")
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""JSON-file backend: no server, no dependencies. Good for tests and offline use."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import pickle
|
|
6
|
+
import shutil
|
|
7
|
+
import time
|
|
8
|
+
import uuid
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Dict, List, Optional
|
|
11
|
+
|
|
12
|
+
from ..errors import BackendError
|
|
13
|
+
from ..settings import Settings
|
|
14
|
+
from .base import Backend, TAG_MODEL_URI
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class LocalBackend(Backend):
|
|
18
|
+
def __init__(self, settings: Settings):
|
|
19
|
+
self.root = Path(settings.tracking_uri or ".authtracker").expanduser()
|
|
20
|
+
(self.root / "runs").mkdir(parents=True, exist_ok=True)
|
|
21
|
+
self._registry_file = self.root / "registry.json"
|
|
22
|
+
|
|
23
|
+
def ping(self) -> None:
|
|
24
|
+
if not self.root.is_dir():
|
|
25
|
+
raise BackendError(f"{self.root} is not a directory")
|
|
26
|
+
|
|
27
|
+
# ---- storage helpers
|
|
28
|
+
def _run_path(self, run_id: str) -> Path:
|
|
29
|
+
return self.root / "runs" / f"{run_id}.json"
|
|
30
|
+
|
|
31
|
+
def _load(self, run_id: str) -> dict:
|
|
32
|
+
try:
|
|
33
|
+
return json.loads(self._run_path(run_id).read_text())
|
|
34
|
+
except FileNotFoundError:
|
|
35
|
+
raise BackendError(f"Run {run_id} not found") from None
|
|
36
|
+
|
|
37
|
+
def _save(self, run: dict) -> None:
|
|
38
|
+
self._run_path(run["run_id"]).write_text(json.dumps(run))
|
|
39
|
+
|
|
40
|
+
def _artifacts(self, run_id: str) -> Path:
|
|
41
|
+
p = self.root / "artifacts" / run_id
|
|
42
|
+
p.mkdir(parents=True, exist_ok=True)
|
|
43
|
+
return p
|
|
44
|
+
|
|
45
|
+
def _registry(self) -> dict:
|
|
46
|
+
if self._registry_file.exists():
|
|
47
|
+
return json.loads(self._registry_file.read_text())
|
|
48
|
+
return {}
|
|
49
|
+
|
|
50
|
+
def _save_registry(self, reg: dict) -> None:
|
|
51
|
+
self._registry_file.write_text(json.dumps(reg))
|
|
52
|
+
|
|
53
|
+
# ---- runs
|
|
54
|
+
def start_run(self, experiment, run_name, tags, parent_id):
|
|
55
|
+
run_id = uuid.uuid4().hex
|
|
56
|
+
self._save({
|
|
57
|
+
"run_id": run_id, "experiment": experiment, "name": run_name,
|
|
58
|
+
"tags": dict(tags), "params": {}, "metrics": {}, "status": "RUNNING",
|
|
59
|
+
"parent": parent_id, "start": time.time(),
|
|
60
|
+
})
|
|
61
|
+
return run_id
|
|
62
|
+
|
|
63
|
+
def end_run(self, run_id, status):
|
|
64
|
+
run = self._load(run_id)
|
|
65
|
+
run["status"] = status
|
|
66
|
+
self._save(run)
|
|
67
|
+
|
|
68
|
+
def set_tags(self, run_id, tags):
|
|
69
|
+
run = self._load(run_id)
|
|
70
|
+
run["tags"].update({k: str(v) for k, v in tags.items()})
|
|
71
|
+
self._save(run)
|
|
72
|
+
|
|
73
|
+
def log_params(self, run_id, params):
|
|
74
|
+
run = self._load(run_id)
|
|
75
|
+
for k, v in params.items():
|
|
76
|
+
if k in run["params"] and run["params"][k] != str(v):
|
|
77
|
+
raise BackendError(f"Param {k!r} already logged with a different value")
|
|
78
|
+
run["params"][k] = str(v)
|
|
79
|
+
self._save(run)
|
|
80
|
+
|
|
81
|
+
def log_metrics(self, run_id, metrics, step):
|
|
82
|
+
run = self._load(run_id)
|
|
83
|
+
for k, v in metrics.items():
|
|
84
|
+
run["metrics"].setdefault(k, []).append([step or 0, float(v)])
|
|
85
|
+
self._save(run)
|
|
86
|
+
|
|
87
|
+
def log_artifact(self, run_id, path):
|
|
88
|
+
shutil.copy(path, self._artifacts(run_id) / Path(path).name)
|
|
89
|
+
|
|
90
|
+
def log_model(self, run_id, model, flavor, **kwargs):
|
|
91
|
+
target = self._artifacts(run_id) / "model.pkl"
|
|
92
|
+
with open(target, "wb") as f:
|
|
93
|
+
pickle.dump(model, f)
|
|
94
|
+
uri = str(target)
|
|
95
|
+
self.set_tags(run_id, {TAG_MODEL_URI: uri})
|
|
96
|
+
return uri
|
|
97
|
+
|
|
98
|
+
def log_model_dir(self, run_id, local_dir):
|
|
99
|
+
target = self._artifacts(run_id) / "model_bundle"
|
|
100
|
+
shutil.copytree(local_dir, target, dirs_exist_ok=True)
|
|
101
|
+
uri = str(target)
|
|
102
|
+
self.set_tags(run_id, {TAG_MODEL_URI: uri})
|
|
103
|
+
return uri
|
|
104
|
+
|
|
105
|
+
def download_bundle(self, name, alias, dst=None):
|
|
106
|
+
import tempfile
|
|
107
|
+
version = self.get_alias(name, alias)
|
|
108
|
+
if version is None:
|
|
109
|
+
return None
|
|
110
|
+
source = Path(self._registry()[name]["versions"][version]["uri"])
|
|
111
|
+
if not (source / "bundle.json").is_file():
|
|
112
|
+
return None
|
|
113
|
+
target = Path(dst or tempfile.mkdtemp(prefix="authtracker-load-"))
|
|
114
|
+
shutil.copytree(source, target, dirs_exist_ok=True)
|
|
115
|
+
return str(target)
|
|
116
|
+
|
|
117
|
+
def find_run(self, experiment, name, tags, parent_id):
|
|
118
|
+
best = None
|
|
119
|
+
for p in (self.root / "runs").glob("*.json"):
|
|
120
|
+
run = json.loads(p.read_text())
|
|
121
|
+
if run["experiment"] != experiment or run["name"] != name or run.get("parent") != parent_id:
|
|
122
|
+
continue
|
|
123
|
+
if any(run["tags"].get(k) != v for k, v in tags.items()):
|
|
124
|
+
continue
|
|
125
|
+
if best is None or run["start"] > best["start"]:
|
|
126
|
+
best = run
|
|
127
|
+
return best["run_id"] if best else None
|
|
128
|
+
|
|
129
|
+
def resume_run(self, run_id, parent_id):
|
|
130
|
+
run = self._load(run_id)
|
|
131
|
+
run["status"] = "RUNNING"
|
|
132
|
+
self._save(run)
|
|
133
|
+
return run_id
|
|
134
|
+
|
|
135
|
+
def last_step(self, run_id):
|
|
136
|
+
steps = [step for points in self._load(run_id)["metrics"].values() for step, _ in points]
|
|
137
|
+
return max(steps) if steps else None
|
|
138
|
+
|
|
139
|
+
def download_artifact(self, run_id, name, folder):
|
|
140
|
+
source = self._artifacts(run_id) / name
|
|
141
|
+
if not source.is_file():
|
|
142
|
+
return None
|
|
143
|
+
return shutil.copy(source, Path(folder) / name)
|
|
144
|
+
|
|
145
|
+
def _summary(self, run: dict) -> Dict[str, Any]:
|
|
146
|
+
return {
|
|
147
|
+
"run_id": run["run_id"], "name": run["name"], "tags": run["tags"],
|
|
148
|
+
"params": run["params"],
|
|
149
|
+
"metrics": {k: v[-1][1] for k, v in run["metrics"].items() if v},
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
def get_run(self, run_id):
|
|
153
|
+
return self._summary(self._load(run_id))
|
|
154
|
+
|
|
155
|
+
def search_runs(self, experiment, tags, metric, mode, limit):
|
|
156
|
+
found = []
|
|
157
|
+
for p in (self.root / "runs").glob("*.json"):
|
|
158
|
+
run = json.loads(p.read_text())
|
|
159
|
+
if run["experiment"] != experiment:
|
|
160
|
+
continue
|
|
161
|
+
if any(run["tags"].get(k) != v for k, v in tags.items()):
|
|
162
|
+
continue
|
|
163
|
+
s = self._summary(run)
|
|
164
|
+
if metric in s["metrics"]:
|
|
165
|
+
found.append(s)
|
|
166
|
+
found.sort(key=lambda r: r["metrics"][metric], reverse=(mode == "max"))
|
|
167
|
+
return found[:limit]
|
|
168
|
+
|
|
169
|
+
# ---- registry
|
|
170
|
+
def register(self, run_id, name, model_uri):
|
|
171
|
+
reg = self._registry()
|
|
172
|
+
entry = reg.setdefault(name, {"versions": {}, "aliases": {}})
|
|
173
|
+
version = str(len(entry["versions"]) + 1)
|
|
174
|
+
entry["versions"][version] = {"run_id": run_id, "uri": model_uri}
|
|
175
|
+
self._save_registry(reg)
|
|
176
|
+
return version
|
|
177
|
+
|
|
178
|
+
def set_alias(self, name, version, alias):
|
|
179
|
+
reg = self._registry()
|
|
180
|
+
if name not in reg or version not in reg[name]["versions"]:
|
|
181
|
+
raise BackendError(f"Model {name!r} version {version} does not exist")
|
|
182
|
+
reg[name]["aliases"][alias] = version
|
|
183
|
+
self._save_registry(reg)
|
|
184
|
+
|
|
185
|
+
def get_alias(self, name, alias):
|
|
186
|
+
return self._registry().get(name, {}).get("aliases", {}).get(alias)
|
|
187
|
+
|
|
188
|
+
def version_run_id(self, name, version):
|
|
189
|
+
try:
|
|
190
|
+
return self._registry()[name]["versions"][str(version)]["run_id"]
|
|
191
|
+
except KeyError:
|
|
192
|
+
raise BackendError(f"Model {name!r} version {version} does not exist") from None
|
|
193
|
+
|
|
194
|
+
def load(self, name, alias):
|
|
195
|
+
version = self.get_alias(name, alias)
|
|
196
|
+
if version is None:
|
|
197
|
+
raise BackendError(f"Model {name!r} has no alias {alias!r}")
|
|
198
|
+
uri = self._registry()[name]["versions"][version]["uri"]
|
|
199
|
+
with open(uri, "rb") as f:
|
|
200
|
+
return pickle.load(f)
|