authtracker 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,16 @@
1
+ """authtracker: variable-configured experiment tracking and model versioning."""
2
+ from .errors import AuthTrackerError, BackendError, PromotionError, UnsupportedError, ValidationError
3
+ from .bundle import load_dir as load_bundle
4
+ from .registry import best_run, load, promote, register, rollback
5
+ from .run import Run, run
6
+ from .simple import current_run, finish, init, log, restore, save, set_tags, start, watch
7
+ from .settings import ConfigError, Settings, configure, get_settings
8
+
9
+ __version__ = "0.0.1"
10
+
11
+ __all__ = [
12
+ "init", "start", "log", "save", "restore", "watch", "finish", "set_tags", "current_run",
13
+ "run", "Run", "register", "promote", "rollback", "load", "load_bundle", "best_run",
14
+ "Settings", "configure", "get_settings", "ConfigError",
15
+ "AuthTrackerError", "ValidationError", "PromotionError", "BackendError", "UnsupportedError",
16
+ ]
@@ -0,0 +1,95 @@
1
+ """Backend lookup: built-ins, runtime registration, and installed plugins.
2
+
3
+ Resolution order for AUTHTRACKER_BACKEND=<name>:
4
+ 1. a backend registered in code with `register_backend`;
5
+ 2. an installed package that declares it in the `authtracker.backends` entry-point group, e.g.
6
+ in that package's pyproject.toml:
7
+
8
+ [project.entry-points."authtracker.backends"]
9
+ wandb = "authtracker_wandb:make_backend" # callable(settings) -> Backend
10
+ """
11
+ from __future__ import annotations
12
+
13
+ from importlib import metadata
14
+ from typing import Callable, Dict, List, Optional
15
+
16
+ from ..errors import BackendError
17
+ from ..settings import Settings
18
+ from .base import BACKEND_API_VERSION, Backend
19
+
20
+ ENTRY_POINT_GROUP = "authtracker.backends"
21
+
22
+ _REGISTRY: Dict[str, Callable[[Settings], Backend]] = {}
23
+
24
+
25
+ def register_backend(name: str, factory: Callable[[Settings], Backend]) -> None:
26
+ _REGISTRY[name] = factory
27
+
28
+
29
+ def _make_local(settings: Settings) -> Backend:
30
+ from .local import LocalBackend
31
+ return LocalBackend(settings)
32
+
33
+
34
+ def _make_mlflow(settings: Settings) -> Backend:
35
+ try:
36
+ from .mlflow_backend import MlflowBackend
37
+ except ImportError as e: # mlflow not installed
38
+ raise BackendError(
39
+ "Backend 'mlflow' needs MLflow. Install one of: pip install 'authtracker[mlflow]' "
40
+ "(full), or pip install 'authtracker[mlflow-skinny]' (light: only sends runs to a server)"
41
+ ) from e
42
+ return MlflowBackend(settings)
43
+
44
+
45
+ def _make_wandb(settings: Settings) -> Backend:
46
+ try:
47
+ from .wandb_backend import WandbBackend
48
+ except ImportError as e: # wandb not installed
49
+ raise BackendError("Backend 'wandb' needs W&B: pip install 'authtracker[wandb]'") from e
50
+ return WandbBackend(settings)
51
+
52
+
53
+ register_backend("mlflow", _make_mlflow)
54
+ register_backend("local", _make_local)
55
+ register_backend("wandb", _make_wandb)
56
+
57
+
58
+ def _entry_points() -> List[metadata.EntryPoint]:
59
+ try:
60
+ return list(metadata.entry_points(group=ENTRY_POINT_GROUP))
61
+ except TypeError: # Python < 3.10
62
+ return list(metadata.entry_points().get(ENTRY_POINT_GROUP, []))
63
+
64
+
65
+ def available_backends() -> List[str]:
66
+ return sorted(set(_REGISTRY) | {ep.name for ep in _entry_points()})
67
+
68
+
69
+ def _find_factory(name: str) -> Optional[Callable[[Settings], Backend]]:
70
+ if name in _REGISTRY:
71
+ return _REGISTRY[name]
72
+ for ep in _entry_points():
73
+ if ep.name == name:
74
+ return ep.load()
75
+ return None
76
+
77
+
78
+ def get_backend(settings: Settings) -> Backend:
79
+ factory = _find_factory(settings.backend)
80
+ if factory is None:
81
+ raise BackendError(
82
+ f"Unknown backend {settings.backend!r}. Available: {available_backends()}")
83
+ backend = factory(settings)
84
+ version = getattr(backend, "api_version", None)
85
+ if version != BACKEND_API_VERSION:
86
+ raise BackendError(
87
+ f"Backend {settings.backend!r} implements backend API version {version}, "
88
+ f"but this authtracker needs version {BACKEND_API_VERSION}")
89
+ if settings.offline_queue and not backend.buffers_offline:
90
+ from .resilient import ResilientBackend
91
+ backend = ResilientBackend(backend, settings)
92
+ return backend
93
+
94
+
95
+ __all__ = ["Backend", "get_backend", "register_backend", "available_backends"]
@@ -0,0 +1,136 @@
1
+ """The only interface the rest of authtracker talks to.
2
+
3
+ To support another tracking platform, subclass `Backend`:
4
+
5
+ * implement the REQUIRED methods (runs, params, metrics, tags, artifacts, queries);
6
+ * optionally implement the model methods (`log_model`) and the registry methods
7
+ (`register`, `set_alias`, `get_alias`, `version_run_id`, `load`). Anything you leave out
8
+ raises a clear BackendError when a user calls it;
9
+ * register it with `register_backend("name", factory)`, or ship it in a separate package via the
10
+ `authtracker.backends` entry-point group (no change to authtracker needed);
11
+ * check it with `authtracker.testing.check_backend(your_backend)`.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ from abc import ABC, abstractmethod
16
+ from typing import Any, Dict, List, Optional
17
+
18
+ from ..errors import BackendError, UnsupportedError
19
+
20
+ # Bump when a change to this interface would break existing backends.
21
+ BACKEND_API_VERSION = 1
22
+
23
+ # Tags the package itself writes.
24
+ TAG_LEVEL = "authtracker.level" # "experiment" (parent) or "run" (child)
25
+ TAG_MODEL_URI = "authtracker.model_uri" # where log_model put the model
26
+ TAG_RUN_ID = "authtracker.run_id" # on model versions: source run
27
+
28
+ _TRANSIENT_NAMES = {"ConnectionError", "Timeout", "ConnectTimeout", "ReadTimeout",
29
+ "TimeoutError", "OperationalError"}
30
+
31
+
32
+ class Backend(ABC):
33
+ api_version = BACKEND_API_VERSION
34
+ # True if the platform already buffers writes while offline (for example W&B's offline mode).
35
+ # authtracker then skips its own retry/queue layer for this backend.
36
+ buffers_offline = False
37
+ # True if the platform already records everything the program prints (W&B does). authtracker
38
+ # then does not copy the console output into an `output.log` artifact itself.
39
+ captures_console = False
40
+
41
+ def is_transient(self, exc: Exception) -> bool:
42
+ """True if `exc` looks like a temporary outage (worth retrying / queuing)."""
43
+ return any(c.__name__ in _TRANSIENT_NAMES for c in type(exc).__mro__)
44
+
45
+ def _unsupported(self, what: str) -> BackendError:
46
+ return UnsupportedError(f"The {type(self).__name__} backend does not support {what}")
47
+
48
+ # ------------------------------------------------------------------ required
49
+ @abstractmethod
50
+ def ping(self) -> None:
51
+ """Raise if the backend is unreachable."""
52
+
53
+ @abstractmethod
54
+ def start_run(self, experiment: str, run_name: str, tags: Dict[str, str],
55
+ parent_id: Optional[str]) -> str:
56
+ """Create a run in `experiment` (the model name) and return its id."""
57
+
58
+ @abstractmethod
59
+ def end_run(self, run_id: str, status: str) -> None:
60
+ """Status is FINISHED, FAILED or KILLED."""
61
+
62
+ @abstractmethod
63
+ def set_tags(self, run_id: str, tags: Dict[str, str]) -> None: ...
64
+
65
+ @abstractmethod
66
+ def log_params(self, run_id: str, params: Dict[str, Any]) -> None: ...
67
+
68
+ @abstractmethod
69
+ def log_metrics(self, run_id: str, metrics: Dict[str, float], step: Optional[int]) -> None: ...
70
+
71
+ @abstractmethod
72
+ def log_artifact(self, run_id: str, path: str) -> None:
73
+ """Store the file at `path` under its own file name."""
74
+
75
+ @abstractmethod
76
+ def get_run(self, run_id: str) -> Dict[str, Any]:
77
+ """Return {'run_id','name','metrics','params','tags'} (metrics = latest values)."""
78
+
79
+ @abstractmethod
80
+ def search_runs(self, experiment: str, tags: Dict[str, str], metric: str,
81
+ mode: str, limit: int) -> List[Dict[str, Any]]:
82
+ """Runs in `experiment` having all `tags` and a value for `metric`, best first
83
+ ('max' = highest first, 'min' = lowest first), at most `limit`, same shape as get_run.
84
+ An unknown experiment gives []."""
85
+
86
+ # ------------------------------------------------------------------ optional: resuming runs
87
+ def find_run(self, experiment: str, name: str, tags: Dict[str, str],
88
+ parent_id: Optional[str]) -> Optional[str]:
89
+ """Id of the newest run in `experiment` that is called `name`, has all `tags` and is a child of
90
+ `parent_id` (None for a top-level run); None if there is no such run."""
91
+ raise self._unsupported("resuming runs")
92
+
93
+ def resume_run(self, run_id: str, parent_id: Optional[str]) -> str:
94
+ """Reopen a finished run so that more can be logged to it, and return its id. If `parent_id` is
95
+ given, that run is the open run it is nested in."""
96
+ raise self._unsupported("resuming runs")
97
+
98
+ def last_step(self, run_id: str) -> Optional[int]:
99
+ """The highest metric step logged to the run so far (None: no metrics, or not known)."""
100
+ return None
101
+
102
+ def download_artifact(self, run_id: str, name: str, folder: str) -> Optional[str]:
103
+ """Copy the file `name` that was saved on the run into `folder` and return its path; None if the run has no such file."""
104
+ raise self._unsupported("downloading files")
105
+
106
+ # ------------------------------------------------------------------ optional: models
107
+ def log_model(self, run_id: str, model: Any, flavor: Optional[str], **kwargs) -> str:
108
+ """Log a model for the run, set the TAG_MODEL_URI tag on it and return the URI."""
109
+ raise self._unsupported("logging models")
110
+
111
+ def log_model_dir(self, run_id: str, local_dir: str) -> str:
112
+ """Store a folder as the run's model (a PyTorch bundle), set TAG_MODEL_URI and return its URI.
113
+ `register` must then be able to make a version from that URI."""
114
+ raise self._unsupported("saving model bundles")
115
+
116
+ def download_bundle(self, name: str, alias: str, dst: Optional[str] = None) -> Optional[str]:
117
+ """If version `alias` of `name` is a model bundle, copy it into a NEW folder and return its
118
+ path (the caller removes it); otherwise return None."""
119
+ return None
120
+
121
+ # ------------------------------------------------------------------ optional: registry
122
+ def register(self, run_id: str, name: str, model_uri: str) -> str:
123
+ """Create a new model version and return its version string."""
124
+ raise self._unsupported("the model registry")
125
+
126
+ def set_alias(self, name: str, version: str, alias: str) -> None:
127
+ raise self._unsupported("the model registry")
128
+
129
+ def get_alias(self, name: str, alias: str) -> Optional[str]:
130
+ raise self._unsupported("the model registry")
131
+
132
+ def version_run_id(self, name: str, version: str) -> str:
133
+ raise self._unsupported("the model registry")
134
+
135
+ def load(self, name: str, alias: str) -> Any:
136
+ raise self._unsupported("the model registry")
@@ -0,0 +1,200 @@
1
+ """JSON-file backend: no server, no dependencies. Good for tests and offline use."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import pickle
6
+ import shutil
7
+ import time
8
+ import uuid
9
+ from pathlib import Path
10
+ from typing import Any, Dict, List, Optional
11
+
12
+ from ..errors import BackendError
13
+ from ..settings import Settings
14
+ from .base import Backend, TAG_MODEL_URI
15
+
16
+
17
+ class LocalBackend(Backend):
18
+ def __init__(self, settings: Settings):
19
+ self.root = Path(settings.tracking_uri or ".authtracker").expanduser()
20
+ (self.root / "runs").mkdir(parents=True, exist_ok=True)
21
+ self._registry_file = self.root / "registry.json"
22
+
23
+ def ping(self) -> None:
24
+ if not self.root.is_dir():
25
+ raise BackendError(f"{self.root} is not a directory")
26
+
27
+ # ---- storage helpers
28
+ def _run_path(self, run_id: str) -> Path:
29
+ return self.root / "runs" / f"{run_id}.json"
30
+
31
+ def _load(self, run_id: str) -> dict:
32
+ try:
33
+ return json.loads(self._run_path(run_id).read_text())
34
+ except FileNotFoundError:
35
+ raise BackendError(f"Run {run_id} not found") from None
36
+
37
+ def _save(self, run: dict) -> None:
38
+ self._run_path(run["run_id"]).write_text(json.dumps(run))
39
+
40
+ def _artifacts(self, run_id: str) -> Path:
41
+ p = self.root / "artifacts" / run_id
42
+ p.mkdir(parents=True, exist_ok=True)
43
+ return p
44
+
45
+ def _registry(self) -> dict:
46
+ if self._registry_file.exists():
47
+ return json.loads(self._registry_file.read_text())
48
+ return {}
49
+
50
+ def _save_registry(self, reg: dict) -> None:
51
+ self._registry_file.write_text(json.dumps(reg))
52
+
53
+ # ---- runs
54
+ def start_run(self, experiment, run_name, tags, parent_id):
55
+ run_id = uuid.uuid4().hex
56
+ self._save({
57
+ "run_id": run_id, "experiment": experiment, "name": run_name,
58
+ "tags": dict(tags), "params": {}, "metrics": {}, "status": "RUNNING",
59
+ "parent": parent_id, "start": time.time(),
60
+ })
61
+ return run_id
62
+
63
+ def end_run(self, run_id, status):
64
+ run = self._load(run_id)
65
+ run["status"] = status
66
+ self._save(run)
67
+
68
+ def set_tags(self, run_id, tags):
69
+ run = self._load(run_id)
70
+ run["tags"].update({k: str(v) for k, v in tags.items()})
71
+ self._save(run)
72
+
73
+ def log_params(self, run_id, params):
74
+ run = self._load(run_id)
75
+ for k, v in params.items():
76
+ if k in run["params"] and run["params"][k] != str(v):
77
+ raise BackendError(f"Param {k!r} already logged with a different value")
78
+ run["params"][k] = str(v)
79
+ self._save(run)
80
+
81
+ def log_metrics(self, run_id, metrics, step):
82
+ run = self._load(run_id)
83
+ for k, v in metrics.items():
84
+ run["metrics"].setdefault(k, []).append([step or 0, float(v)])
85
+ self._save(run)
86
+
87
+ def log_artifact(self, run_id, path):
88
+ shutil.copy(path, self._artifacts(run_id) / Path(path).name)
89
+
90
+ def log_model(self, run_id, model, flavor, **kwargs):
91
+ target = self._artifacts(run_id) / "model.pkl"
92
+ with open(target, "wb") as f:
93
+ pickle.dump(model, f)
94
+ uri = str(target)
95
+ self.set_tags(run_id, {TAG_MODEL_URI: uri})
96
+ return uri
97
+
98
+ def log_model_dir(self, run_id, local_dir):
99
+ target = self._artifacts(run_id) / "model_bundle"
100
+ shutil.copytree(local_dir, target, dirs_exist_ok=True)
101
+ uri = str(target)
102
+ self.set_tags(run_id, {TAG_MODEL_URI: uri})
103
+ return uri
104
+
105
+ def download_bundle(self, name, alias, dst=None):
106
+ import tempfile
107
+ version = self.get_alias(name, alias)
108
+ if version is None:
109
+ return None
110
+ source = Path(self._registry()[name]["versions"][version]["uri"])
111
+ if not (source / "bundle.json").is_file():
112
+ return None
113
+ target = Path(dst or tempfile.mkdtemp(prefix="authtracker-load-"))
114
+ shutil.copytree(source, target, dirs_exist_ok=True)
115
+ return str(target)
116
+
117
+ def find_run(self, experiment, name, tags, parent_id):
118
+ best = None
119
+ for p in (self.root / "runs").glob("*.json"):
120
+ run = json.loads(p.read_text())
121
+ if run["experiment"] != experiment or run["name"] != name or run.get("parent") != parent_id:
122
+ continue
123
+ if any(run["tags"].get(k) != v for k, v in tags.items()):
124
+ continue
125
+ if best is None or run["start"] > best["start"]:
126
+ best = run
127
+ return best["run_id"] if best else None
128
+
129
+ def resume_run(self, run_id, parent_id):
130
+ run = self._load(run_id)
131
+ run["status"] = "RUNNING"
132
+ self._save(run)
133
+ return run_id
134
+
135
+ def last_step(self, run_id):
136
+ steps = [step for points in self._load(run_id)["metrics"].values() for step, _ in points]
137
+ return max(steps) if steps else None
138
+
139
+ def download_artifact(self, run_id, name, folder):
140
+ source = self._artifacts(run_id) / name
141
+ if not source.is_file():
142
+ return None
143
+ return shutil.copy(source, Path(folder) / name)
144
+
145
+ def _summary(self, run: dict) -> Dict[str, Any]:
146
+ return {
147
+ "run_id": run["run_id"], "name": run["name"], "tags": run["tags"],
148
+ "params": run["params"],
149
+ "metrics": {k: v[-1][1] for k, v in run["metrics"].items() if v},
150
+ }
151
+
152
+ def get_run(self, run_id):
153
+ return self._summary(self._load(run_id))
154
+
155
+ def search_runs(self, experiment, tags, metric, mode, limit):
156
+ found = []
157
+ for p in (self.root / "runs").glob("*.json"):
158
+ run = json.loads(p.read_text())
159
+ if run["experiment"] != experiment:
160
+ continue
161
+ if any(run["tags"].get(k) != v for k, v in tags.items()):
162
+ continue
163
+ s = self._summary(run)
164
+ if metric in s["metrics"]:
165
+ found.append(s)
166
+ found.sort(key=lambda r: r["metrics"][metric], reverse=(mode == "max"))
167
+ return found[:limit]
168
+
169
+ # ---- registry
170
+ def register(self, run_id, name, model_uri):
171
+ reg = self._registry()
172
+ entry = reg.setdefault(name, {"versions": {}, "aliases": {}})
173
+ version = str(len(entry["versions"]) + 1)
174
+ entry["versions"][version] = {"run_id": run_id, "uri": model_uri}
175
+ self._save_registry(reg)
176
+ return version
177
+
178
+ def set_alias(self, name, version, alias):
179
+ reg = self._registry()
180
+ if name not in reg or version not in reg[name]["versions"]:
181
+ raise BackendError(f"Model {name!r} version {version} does not exist")
182
+ reg[name]["aliases"][alias] = version
183
+ self._save_registry(reg)
184
+
185
+ def get_alias(self, name, alias):
186
+ return self._registry().get(name, {}).get("aliases", {}).get(alias)
187
+
188
+ def version_run_id(self, name, version):
189
+ try:
190
+ return self._registry()[name]["versions"][str(version)]["run_id"]
191
+ except KeyError:
192
+ raise BackendError(f"Model {name!r} version {version} does not exist") from None
193
+
194
+ def load(self, name, alias):
195
+ version = self.get_alias(name, alias)
196
+ if version is None:
197
+ raise BackendError(f"Model {name!r} has no alias {alias!r}")
198
+ uri = self._registry()[name]["versions"][version]["uri"]
199
+ with open(uri, "rb") as f:
200
+ return pickle.load(f)