bladekit 4.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bladekit/__init__.py +219 -0
- bladekit/_app.py +3241 -0
- bladekit/_banner.py +640 -0
- bladekit/_batch.py +150 -0
- bladekit/_bucket.py +211 -0
- bladekit/_build.py +337 -0
- bladekit/_bundle.py +459 -0
- bladekit/_client.py +1152 -0
- bladekit/_cls.py +574 -0
- bladekit/_constants.py +121 -0
- bladekit/_cron.py +88 -0
- bladekit/_dict.py +174 -0
- bladekit/_env.py +98 -0
- bladekit/_image.py +577 -0
- bladekit/_loader.py +79 -0
- bladekit/_map.py +555 -0
- bladekit/_preempt.py +94 -0
- bladekit/_queue.py +137 -0
- bladekit/_render.py +683 -0
- bladekit/_runner_api.py +148 -0
- bladekit/_sandbox.py +1051 -0
- bladekit/_secret.py +52 -0
- bladekit/_serialize.py +88 -0
- bladekit/_serve.py +188 -0
- bladekit/_volume.py +232 -0
- bladekit/_webhook.py +90 -0
- bladekit/cli.py +2282 -0
- bladekit/config.py +181 -0
- bladekit/exceptions.py +544 -0
- bladekit-4.5.0.dist-info/METADATA +373 -0
- bladekit-4.5.0.dist-info/RECORD +33 -0
- bladekit-4.5.0.dist-info/WHEEL +4 -0
- bladekit-4.5.0.dist-info/entry_points.txt +3 -0
bladekit/_secret.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""``bladekit.Secret`` — a named secret reference.
|
|
2
|
+
|
|
3
|
+
A ``Secret`` is just a *reference by name*; values never live in the SDK object.
|
|
4
|
+
It is passed to ``@app.function(secrets=[bladekit.Secret.from_name("hf-token")])``
|
|
5
|
+
and resolved by the control plane at run creation (404 SECRET_NOT_FOUND if
|
|
6
|
+
unknown) and again at scheduling (contracts/openapi.yaml, conventions.md §11).
|
|
7
|
+
|
|
8
|
+
Creating / replacing secret *values* is done by the CLI (``blade secret set``),
|
|
9
|
+
which reads the value from stdin or ``--from-env`` and NEVER from a positional
|
|
10
|
+
argument (argv leaks into ``ps``).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
|
|
17
|
+
from . import exceptions as exc
|
|
18
|
+
|
|
19
|
+
# conventions.md §6: ^[a-z0-9][a-z0-9._-]{0,63}$
|
|
20
|
+
_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Secret:
|
|
24
|
+
"""Reference to a named secret stored (encrypted) on the control plane."""
|
|
25
|
+
|
|
26
|
+
__slots__ = ("name",)
|
|
27
|
+
|
|
28
|
+
def __init__(self, name: str) -> None:
|
|
29
|
+
# Private: use :meth:`from_name`.
|
|
30
|
+
self.name = name
|
|
31
|
+
|
|
32
|
+
@classmethod
|
|
33
|
+
def from_name(cls, name: str) -> "Secret":
|
|
34
|
+
"""Reference an existing secret by name (resolved server-side at run time)."""
|
|
35
|
+
if not isinstance(name, str) or not _NAME_RE.match(name):
|
|
36
|
+
raise exc.InvalidArgumentError(
|
|
37
|
+
"INVALID_ARGUMENT",
|
|
38
|
+
f"invalid secret name {name!r}; must match "
|
|
39
|
+
"'^[a-z0-9][a-z0-9._-]{0,63}$'",
|
|
40
|
+
)
|
|
41
|
+
return cls(name)
|
|
42
|
+
|
|
43
|
+
def __repr__(self) -> str:
|
|
44
|
+
return f"<bladekit.Secret name={self.name!r}>"
|
|
45
|
+
|
|
46
|
+
def __eq__(self, other: object) -> bool:
|
|
47
|
+
if not isinstance(other, Secret):
|
|
48
|
+
return NotImplemented
|
|
49
|
+
return self.name == other.name
|
|
50
|
+
|
|
51
|
+
def __hash__(self) -> int:
|
|
52
|
+
return hash(self.name)
|
bladekit/_serialize.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""cloudpickle serialization helpers and the inline/object-store boundary.
|
|
2
|
+
|
|
3
|
+
conventions.md §8: the format is ``cloudpickle``; payloads <= 1 MiB raw go inline
|
|
4
|
+
as base64, larger ones go through a presigned ``pl_`` payload.
|
|
5
|
+
|
|
6
|
+
``cloudpickle`` is imported lazily inside functions so that ``import bladekit`` stays
|
|
7
|
+
cheap (test requirement: no heavy top-level import).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import base64
|
|
13
|
+
import hashlib
|
|
14
|
+
from typing import Any, Tuple
|
|
15
|
+
|
|
16
|
+
from ._constants import INLINE_PAYLOAD_MAX_BYTES, PAYLOAD_FORMAT
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def cloudpickle_version() -> str:
|
|
20
|
+
"""Return cloudpickle version truncated to ``major.minor`` (handshake §5)."""
|
|
21
|
+
import cloudpickle # local import keeps top-level import light
|
|
22
|
+
|
|
23
|
+
parts = cloudpickle.__version__.split(".")
|
|
24
|
+
return f"{parts[0]}.{parts[1]}"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def dumps(obj: Any) -> bytes:
|
|
28
|
+
import cloudpickle
|
|
29
|
+
|
|
30
|
+
return cloudpickle.dumps(obj)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def loads(data: bytes) -> Any:
|
|
34
|
+
import cloudpickle
|
|
35
|
+
|
|
36
|
+
return cloudpickle.loads(data)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def sha256_hex(data: bytes) -> str:
|
|
40
|
+
return hashlib.sha256(data).hexdigest()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def dumps_args(args: tuple, kwargs: dict) -> bytes:
|
|
44
|
+
"""Serialize the ``(args, kwargs)`` tuple — the unit a call carries."""
|
|
45
|
+
return dumps((args, kwargs))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def make_payload_in(raw: bytes) -> Tuple[dict, bytes | None]:
|
|
49
|
+
"""Build a ``PayloadIn`` dict from raw serialized bytes.
|
|
50
|
+
|
|
51
|
+
Returns ``(payload_in, object_bytes)``. When the payload exceeds the inline
|
|
52
|
+
ceiling, ``payload_in`` carries no ``inline_b64`` and ``object_bytes`` holds
|
|
53
|
+
the bytes to PUT to a presigned URL (the caller declares the ``pl_`` payload
|
|
54
|
+
and fills ``payload_id``). Otherwise ``object_bytes`` is ``None``.
|
|
55
|
+
"""
|
|
56
|
+
body: dict[str, Any] = {
|
|
57
|
+
"format": PAYLOAD_FORMAT,
|
|
58
|
+
"size_bytes": len(raw),
|
|
59
|
+
"sha256": sha256_hex(raw),
|
|
60
|
+
}
|
|
61
|
+
if len(raw) <= INLINE_PAYLOAD_MAX_BYTES:
|
|
62
|
+
body["inline_b64"] = base64.b64encode(raw).decode("ascii")
|
|
63
|
+
return body, None
|
|
64
|
+
return body, raw
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def decode_payload_out(payload: dict, *, fetch=None) -> bytes:
|
|
68
|
+
"""Materialize a ``PayloadOut`` to raw bytes, verifying the sha256.
|
|
69
|
+
|
|
70
|
+
``fetch(url) -> bytes`` is used when the payload lives in the object store
|
|
71
|
+
(``get_url`` present). Injecting it keeps this module free of any HTTP
|
|
72
|
+
dependency and trivially testable.
|
|
73
|
+
"""
|
|
74
|
+
inline = payload.get("inline_b64")
|
|
75
|
+
if inline is not None:
|
|
76
|
+
raw = base64.b64decode(inline)
|
|
77
|
+
else:
|
|
78
|
+
get_url = payload.get("get_url")
|
|
79
|
+
if not get_url or fetch is None:
|
|
80
|
+
raise ValueError(
|
|
81
|
+
"payload has no inline_b64 and no usable get_url to download from"
|
|
82
|
+
)
|
|
83
|
+
raw = fetch(get_url)
|
|
84
|
+
|
|
85
|
+
expected = payload.get("sha256")
|
|
86
|
+
if expected and sha256_hex(raw) != expected:
|
|
87
|
+
raise ValueError("payload checksum mismatch")
|
|
88
|
+
return raw
|
bladekit/_serve.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""``blade serve script.py`` — dev loop: watch files, hot-reload the bundle.
|
|
2
|
+
|
|
3
|
+
Flow (runner-protocol.md §9):
|
|
4
|
+
|
|
5
|
+
1. Load the script, create the run with ``serve=true`` (so the container never
|
|
6
|
+
drains on idle), via the first remote invocation in the entrypoint.
|
|
7
|
+
2. Stream the run's logs continuously in the background.
|
|
8
|
+
3. Watch the bundle root for file changes (``watchfiles``). On each change:
|
|
9
|
+
re-bundle (deduplicated by sha256 → usually instant, no upload), POST
|
|
10
|
+
``/v1/runs/{id}/bundle``, print ``reloaded in X.Xs``.
|
|
11
|
+
4. Ctrl-C → cancel the run cleanly and exit.
|
|
12
|
+
|
|
13
|
+
``watchfiles`` is imported lazily *inside* this module's functions so that
|
|
14
|
+
``import bladekit`` (and ``blade --help``) never pulls it in.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import threading
|
|
20
|
+
import time
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, Optional
|
|
23
|
+
|
|
24
|
+
# ``watchfiles`` is imported lazily here so that the *top-level* ``import bladekit``
|
|
25
|
+
# (and ``blade --help``) never pulls it in — _serve is only imported on demand
|
|
26
|
+
# by the CLI's ``serve`` command. The module-level name allows test patching via
|
|
27
|
+
# ``patch("bladekit._serve.watch", ...)``.
|
|
28
|
+
try:
|
|
29
|
+
from watchfiles import watch
|
|
30
|
+
except ImportError: # pragma: no cover
|
|
31
|
+
watch = None # type: ignore[assignment]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def serve(client: Any, forge_app: Any, entrypoint: Any, bundle_root: Path) -> int:
|
|
35
|
+
"""Run the serve dev loop. Returns the process exit code."""
|
|
36
|
+
from . import _app as app_mod
|
|
37
|
+
from . import exceptions as exc
|
|
38
|
+
from ._render import console, err_console
|
|
39
|
+
|
|
40
|
+
# Mark every function on the app as serve=true so its run never drains on
|
|
41
|
+
# idle — the reload needs a live container (runner-protocol.md §9).
|
|
42
|
+
for fn in forge_app.functions.values():
|
|
43
|
+
fn.serve = True
|
|
44
|
+
for cls in forge_app.classes.values():
|
|
45
|
+
cls._serve = True # consumed when the method's Function is built
|
|
46
|
+
# Clear any Function objects already built with serve=False (e.g. from
|
|
47
|
+
# a prior invocation or an eager call during module load). The next
|
|
48
|
+
# _function_for() call will rebuild them with serve=True.
|
|
49
|
+
cls._functions.clear()
|
|
50
|
+
|
|
51
|
+
stop_event = threading.Event()
|
|
52
|
+
streamed = {"run_id": None}
|
|
53
|
+
|
|
54
|
+
def _log_pump():
|
|
55
|
+
from .cli import _stream_run_logs
|
|
56
|
+
|
|
57
|
+
while not stop_event.is_set():
|
|
58
|
+
run_ids = list(app_mod._SESSION.open_run_ids)
|
|
59
|
+
if run_ids:
|
|
60
|
+
streamed["run_id"] = run_ids[0]
|
|
61
|
+
break
|
|
62
|
+
stop_event.wait(0.1)
|
|
63
|
+
rid = streamed["run_id"]
|
|
64
|
+
if rid:
|
|
65
|
+
_stream_run_logs(client, rid, stop_event)
|
|
66
|
+
|
|
67
|
+
log_thread = threading.Thread(target=_log_pump, daemon=True)
|
|
68
|
+
log_thread.start()
|
|
69
|
+
|
|
70
|
+
# Watch the bundle CONCURRENTLY with the entrypoint. The entrypoint may be a
|
|
71
|
+
# long-running dev loop (`for _ in range(N): fn.remote(); sleep`) — the whole
|
|
72
|
+
# point of serve is to edit code *while it runs* and have the next call pick
|
|
73
|
+
# up the new bundle. A sequential "run entrypoint to completion, then watch"
|
|
74
|
+
# would never reload such an entrypoint. The watcher waits for the run to
|
|
75
|
+
# appear (created by the first .remote() in the entrypoint), then hot-reloads
|
|
76
|
+
# on every change until stop_event.
|
|
77
|
+
watch_started = threading.Event()
|
|
78
|
+
|
|
79
|
+
def _watch_pump():
|
|
80
|
+
run_id = None
|
|
81
|
+
while not stop_event.is_set():
|
|
82
|
+
run_ids = list(app_mod._SESSION.open_run_ids)
|
|
83
|
+
if run_ids:
|
|
84
|
+
run_id = run_ids[0]
|
|
85
|
+
break
|
|
86
|
+
stop_event.wait(0.1)
|
|
87
|
+
if run_id is None:
|
|
88
|
+
return
|
|
89
|
+
console.print(
|
|
90
|
+
f"[green]serving[/green] run {run_id} — watching {bundle_root} "
|
|
91
|
+
"for changes. Ctrl-C to stop."
|
|
92
|
+
)
|
|
93
|
+
watch_started.set()
|
|
94
|
+
try:
|
|
95
|
+
_watch_loop(client, forge_app, run_id, bundle_root, stop_event)
|
|
96
|
+
except exc.FabricError as e: # pragma: no cover - surfaced, non-fatal
|
|
97
|
+
err_console.print(f"[red]watch error[/red]: {e}")
|
|
98
|
+
|
|
99
|
+
watch_thread = threading.Thread(target=_watch_pump, daemon=True)
|
|
100
|
+
watch_thread.start()
|
|
101
|
+
|
|
102
|
+
exit_code = 0
|
|
103
|
+
try:
|
|
104
|
+
# First invocation: creates the serve run + the first call. May return
|
|
105
|
+
# quickly (one call) or loop for a while (a dev iterating); the watcher
|
|
106
|
+
# above is already running either way.
|
|
107
|
+
console.print("[cyan]blade serve[/cyan]: starting run (serve mode)…")
|
|
108
|
+
entrypoint()
|
|
109
|
+
run_ids = list(app_mod._SESSION.open_run_ids)
|
|
110
|
+
if not run_ids:
|
|
111
|
+
err_console.print(
|
|
112
|
+
"[yellow]no run was created by the entrypoint; nothing to "
|
|
113
|
+
"serve.[/yellow]"
|
|
114
|
+
)
|
|
115
|
+
stop_event.set()
|
|
116
|
+
return 0
|
|
117
|
+
# The entrypoint returned (or never blocked): keep serving until Ctrl-C
|
|
118
|
+
# so the dev can still edit + reload after a quick entrypoint.
|
|
119
|
+
while not stop_event.is_set():
|
|
120
|
+
stop_event.wait(0.5)
|
|
121
|
+
except KeyboardInterrupt:
|
|
122
|
+
console.print("\n[magenta]Ctrl-C — stopping serve…[/magenta]")
|
|
123
|
+
exit_code = 130
|
|
124
|
+
except exc.FabricError as e:
|
|
125
|
+
err_console.print(f"[red]serve error[/red]: {e}")
|
|
126
|
+
exit_code = 1
|
|
127
|
+
finally:
|
|
128
|
+
# Cancel every run this serve session opened (clean shutdown).
|
|
129
|
+
# Only swallow ConflictError (RUN_NOT_CANCELLABLE = already terminal).
|
|
130
|
+
# Network or auth errors are caught by the broad except below so they
|
|
131
|
+
# don't abort the rest of the cleanup, but they are not silenced for
|
|
132
|
+
# runs that follow — the broad catch at the finally level handles them.
|
|
133
|
+
for rid in list(app_mod._SESSION.open_run_ids):
|
|
134
|
+
try:
|
|
135
|
+
client.cancel_run(rid)
|
|
136
|
+
except exc.ConflictError:
|
|
137
|
+
# Run already terminal — expected, ignore.
|
|
138
|
+
pass
|
|
139
|
+
except Exception: # noqa: BLE001
|
|
140
|
+
# Network / auth failure on cancel: don't abort cleanup of
|
|
141
|
+
# remaining runs; the run will time out server-side.
|
|
142
|
+
pass
|
|
143
|
+
stop_event.set()
|
|
144
|
+
log_thread.join(timeout=3.0)
|
|
145
|
+
watch_thread.join(timeout=3.0)
|
|
146
|
+
return exit_code
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _watch_loop(
|
|
150
|
+
client: Any,
|
|
151
|
+
forge_app: Any,
|
|
152
|
+
run_id: str,
|
|
153
|
+
bundle_root: Path,
|
|
154
|
+
stop_event: threading.Event,
|
|
155
|
+
) -> None:
|
|
156
|
+
"""Watch ``bundle_root`` and hot-reload the bundle on each change."""
|
|
157
|
+
from . import exceptions as exc
|
|
158
|
+
from ._render import console, err_console
|
|
159
|
+
|
|
160
|
+
if watch is None: # pragma: no cover
|
|
161
|
+
err_console.print(
|
|
162
|
+
"[red]blade serve needs 'watchfiles'[/red]: pip install "
|
|
163
|
+
"'watchfiles'."
|
|
164
|
+
)
|
|
165
|
+
return
|
|
166
|
+
|
|
167
|
+
last_sha: Optional[str] = forge_app._bundle_sha256
|
|
168
|
+
for _changes in watch(str(bundle_root), stop_event=stop_event):
|
|
169
|
+
if stop_event.is_set():
|
|
170
|
+
break
|
|
171
|
+
t0 = time.perf_counter()
|
|
172
|
+
try:
|
|
173
|
+
new_id, new_sha = forge_app._rebuild_bundle()
|
|
174
|
+
except exc.FabricError as e:
|
|
175
|
+
err_console.print(f"[red]re-bundle failed[/red]: {e}")
|
|
176
|
+
continue
|
|
177
|
+
if new_sha == last_sha:
|
|
178
|
+
# Editor touched a file without changing content (or only ignored
|
|
179
|
+
# files changed): nothing to push.
|
|
180
|
+
continue
|
|
181
|
+
try:
|
|
182
|
+
client.update_run_bundle(run_id, new_id)
|
|
183
|
+
except exc.FabricError as e:
|
|
184
|
+
err_console.print(f"[red]reload failed[/red]: {e}")
|
|
185
|
+
continue
|
|
186
|
+
last_sha = new_sha
|
|
187
|
+
elapsed = time.perf_counter() - t0
|
|
188
|
+
console.print(f"[green]reloaded in {elapsed:.1f}s[/green] [dim]({new_sha[:12]}…)[/dim]")
|
bladekit/_volume.py
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
"""``bladekit.Volume`` — a named network volume reference, with in-container ``.commit()``.
|
|
2
|
+
|
|
3
|
+
Two roles, one object:
|
|
4
|
+
|
|
5
|
+
1. **Client-side reference**: ``bladekit.Volume.from_name("datasets")`` passed via
|
|
6
|
+
``@app.function(volumes={"/data": vol})``. ``create_if_missing=True`` makes
|
|
7
|
+
the run-creation path create-or-get the volume (openapi createVolume is
|
|
8
|
+
idempotent by name). The mount generation is resolved at scheduling, frozen
|
|
9
|
+
on the run (a concurrent commit never changes a scheduled container's view).
|
|
10
|
+
|
|
11
|
+
2. **In-container handle**: inside an AI Fabric container the user calls
|
|
12
|
+
``vol.commit()`` to publish local writes as a new generation. The SDK detects
|
|
13
|
+
the container context via ``FABRIC_VOLUMES`` and drives the commit through the
|
|
14
|
+
*runner-internal* API with the run token (the runner never talks to the
|
|
15
|
+
agent — invariant kept, runner-protocol.md §11).
|
|
16
|
+
|
|
17
|
+
Resolution rule (§11): ``commit()`` finds the matching mount in ``FABRIC_VOLUMES``
|
|
18
|
+
by name; if absent it raises ``bladekit.VolumeError`` locally rather than letting
|
|
19
|
+
the server answer 404. ``read_only`` mounts raise immediately.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import re
|
|
26
|
+
import time
|
|
27
|
+
from typing import Any, Dict, List, Optional
|
|
28
|
+
|
|
29
|
+
from . import exceptions as exc
|
|
30
|
+
|
|
31
|
+
_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$")
|
|
32
|
+
|
|
33
|
+
# Commit poll backoff (runner-protocol.md §11: 1 s → 5 s).
|
|
34
|
+
_COMMIT_POLL_INITIAL = 1.0
|
|
35
|
+
_COMMIT_POLL_MAX = 5.0
|
|
36
|
+
_COMMIT_TERMINAL = frozenset({"committed", "failed"})
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Volume:
|
|
40
|
+
"""Reference to a named network volume (and in-container commit handle)."""
|
|
41
|
+
|
|
42
|
+
__slots__ = (
|
|
43
|
+
"name",
|
|
44
|
+
"_create_if_missing",
|
|
45
|
+
"_auto_commit_on_ready",
|
|
46
|
+
"_auto_commit_every",
|
|
47
|
+
"_read_only",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def __init__(
|
|
51
|
+
self,
|
|
52
|
+
name: str,
|
|
53
|
+
*,
|
|
54
|
+
create_if_missing: bool = False,
|
|
55
|
+
auto_commit_on_ready: bool = False,
|
|
56
|
+
auto_commit_every: int = 0,
|
|
57
|
+
read_only: bool = False,
|
|
58
|
+
) -> None:
|
|
59
|
+
# Private: use :meth:`from_name`.
|
|
60
|
+
self.name = name
|
|
61
|
+
self._create_if_missing = create_if_missing
|
|
62
|
+
self._auto_commit_on_ready = auto_commit_on_ready
|
|
63
|
+
self._auto_commit_every = auto_commit_every
|
|
64
|
+
self._read_only = read_only
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def from_name(
|
|
68
|
+
cls,
|
|
69
|
+
name: str,
|
|
70
|
+
*,
|
|
71
|
+
create_if_missing: bool = False,
|
|
72
|
+
auto_commit_on_ready: bool = False,
|
|
73
|
+
auto_commit_every: int = 0,
|
|
74
|
+
read_only: bool = False,
|
|
75
|
+
) -> "Volume":
|
|
76
|
+
"""Reference a volume by name.
|
|
77
|
+
|
|
78
|
+
``create_if_missing=True`` creates it (generation 0, empty) if it does
|
|
79
|
+
not exist yet, the first time the run that mounts it is created.
|
|
80
|
+
|
|
81
|
+
``auto_commit_on_ready=True`` (deployed ``web_server``/``asgi_app``
|
|
82
|
+
endpoints only) commits the mount automatically once the replica
|
|
83
|
+
reports serving-ready — no ``vol.commit()`` needed in user code. Made
|
|
84
|
+
for server warm-up caches (e.g. vLLM ``VLLM_CACHE_ROOT``): whatever the
|
|
85
|
+
server wrote under the mount during startup is published for the next
|
|
86
|
+
cold start, on any node. Best-effort and non-blocking: the commit runs
|
|
87
|
+
after traffic starts flowing, and a failure never affects the replica.
|
|
88
|
+
|
|
89
|
+
``auto_commit_every=N`` (seconds, same endpoint kinds) commits the mount
|
|
90
|
+
every N seconds from container boot, WITHOUT waiting for serving-ready.
|
|
91
|
+
Use it when the thing being cached is what makes the boot slow: a first
|
|
92
|
+
``torch.compile`` that outruns ``startup_timeout`` kills the replica
|
|
93
|
+
before ``auto_commit_on_ready`` can fire, so the cache it just built is
|
|
94
|
+
lost and the next replica rebuilds it and fails identically. A periodic
|
|
95
|
+
commit breaks that loop because it does not wait for the boot to
|
|
96
|
+
succeed. Minimum 60; 0 (the default) is off. Combines with
|
|
97
|
+
``auto_commit_on_ready``.
|
|
98
|
+
"""
|
|
99
|
+
if not isinstance(name, str) or not _NAME_RE.match(name):
|
|
100
|
+
raise exc.InvalidArgumentError(
|
|
101
|
+
"INVALID_ARGUMENT",
|
|
102
|
+
f"invalid volume name {name!r}; must match "
|
|
103
|
+
"'^[a-z0-9][a-z0-9._-]{0,63}$'",
|
|
104
|
+
)
|
|
105
|
+
if not isinstance(auto_commit_every, int) or isinstance(
|
|
106
|
+
auto_commit_every, bool
|
|
107
|
+
):
|
|
108
|
+
raise exc.InvalidArgumentError(
|
|
109
|
+
"INVALID_ARGUMENT", "auto_commit_every must be an integer"
|
|
110
|
+
)
|
|
111
|
+
if auto_commit_every != 0 and auto_commit_every < 60:
|
|
112
|
+
raise exc.InvalidArgumentError(
|
|
113
|
+
"INVALID_ARGUMENT",
|
|
114
|
+
"auto_commit_every must be 0 (off) or at least 60 seconds",
|
|
115
|
+
)
|
|
116
|
+
return cls(
|
|
117
|
+
name,
|
|
118
|
+
create_if_missing=create_if_missing,
|
|
119
|
+
auto_commit_on_ready=auto_commit_on_ready,
|
|
120
|
+
auto_commit_every=auto_commit_every,
|
|
121
|
+
read_only=read_only,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
def __repr__(self) -> str:
|
|
125
|
+
return f"<bladekit.Volume name={self.name!r}>"
|
|
126
|
+
|
|
127
|
+
def __eq__(self, other: object) -> bool:
|
|
128
|
+
if not isinstance(other, Volume):
|
|
129
|
+
return NotImplemented
|
|
130
|
+
return self.name == other.name
|
|
131
|
+
|
|
132
|
+
def __hash__(self) -> int:
|
|
133
|
+
return hash(self.name)
|
|
134
|
+
|
|
135
|
+
# -- in-container commit (runner-protocol.md §11) -----------------------
|
|
136
|
+
|
|
137
|
+
# Default poll timeout (seconds). Large commits — model weights pushed via
|
|
138
|
+
# S3 multipart — legitimately run well past 10 minutes, so the client must
|
|
139
|
+
# not give up early; the server-side stall watcher (FORGE_COMMIT_STALL_AFTER)
|
|
140
|
+
# is what actually bounds a genuinely stuck commit. Override per-call via
|
|
141
|
+
# ``timeout=`` for very large volumes.
|
|
142
|
+
_COMMIT_DEFAULT_TIMEOUT = 3600.0
|
|
143
|
+
|
|
144
|
+
def commit(self, *, poll: bool = True, timeout: Optional[float] = _COMMIT_DEFAULT_TIMEOUT) -> int:
|
|
145
|
+
"""Publish this container's local writes as a new volume generation.
|
|
146
|
+
|
|
147
|
+
Only valid inside an AI Fabric container (``FABRIC_VOLUMES`` or ``FORGE_VOLUMES`` set). Resolves the
|
|
148
|
+
mount by name, POSTs the runner-internal commit, then polls until the
|
|
149
|
+
commit is ``committed`` (returns the new generation) or ``failed``
|
|
150
|
+
(raises :class:`bladekit.VolumeCommitError`).
|
|
151
|
+
|
|
152
|
+
``timeout`` caps the total poll time in seconds (default 600 s, matching
|
|
153
|
+
the server-side commit watcher). Pass ``None`` for no timeout (not
|
|
154
|
+
recommended in production).
|
|
155
|
+
|
|
156
|
+
A failed commit is never fatal: the container keeps running and the call
|
|
157
|
+
may retry (runner-protocol.md §11).
|
|
158
|
+
"""
|
|
159
|
+
mount = _resolve_mount(self.name)
|
|
160
|
+
if mount is None:
|
|
161
|
+
raise exc.VolumeError(
|
|
162
|
+
f"volume {self.name!r} is not mounted in this container; "
|
|
163
|
+
"pass it via volumes={'/path': vol} on @app.function."
|
|
164
|
+
)
|
|
165
|
+
if mount.get("read_only"):
|
|
166
|
+
raise exc.VolumeCommitError(
|
|
167
|
+
"VOLUME_READ_ONLY",
|
|
168
|
+
f"volume {self.name!r} is mounted read-only; commit() is refused.",
|
|
169
|
+
)
|
|
170
|
+
volume_id = mount["volume_id"]
|
|
171
|
+
|
|
172
|
+
from ._runner_api import RunnerApiClient
|
|
173
|
+
|
|
174
|
+
client = RunnerApiClient.from_env()
|
|
175
|
+
commit = client.commit_volume(volume_id)
|
|
176
|
+
commit_id = commit["commit_id"]
|
|
177
|
+
if not poll:
|
|
178
|
+
return commit # type: ignore[return-value]
|
|
179
|
+
|
|
180
|
+
deadline = None if timeout is None else time.monotonic() + timeout
|
|
181
|
+
interval = _COMMIT_POLL_INITIAL
|
|
182
|
+
while True:
|
|
183
|
+
state = commit.get("state")
|
|
184
|
+
if state in _COMMIT_TERMINAL:
|
|
185
|
+
if state == "committed":
|
|
186
|
+
return int(commit["generation"])
|
|
187
|
+
err = commit.get("error") or {}
|
|
188
|
+
raise exc.VolumeCommitError(
|
|
189
|
+
str(err.get("code", "VOLUME_COMMIT_FAILED")),
|
|
190
|
+
str(err.get("message", "volume commit failed")),
|
|
191
|
+
)
|
|
192
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
193
|
+
raise exc.VolumeCommitError(
|
|
194
|
+
"VOLUME_COMMIT_FAILED",
|
|
195
|
+
f"timed out after {timeout}s waiting for commit {commit_id}",
|
|
196
|
+
)
|
|
197
|
+
time.sleep(interval)
|
|
198
|
+
interval = min(interval * 1.5, _COMMIT_POLL_MAX)
|
|
199
|
+
commit = client.get_volume_commit(commit_id)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _mounted_volumes() -> List[Dict[str, Any]]:
|
|
203
|
+
"""Parse ``FABRIC_VOLUMES`` / ``FORGE_VOLUMES`` (JSON list) — empty/absent ⇒ no volumes.
|
|
204
|
+
|
|
205
|
+
An unparseable value is logged as a warning rather than silently returning
|
|
206
|
+
[] — an injection bug (platform side) would otherwise surface only as a
|
|
207
|
+
confusing "volume not mounted" error from commit().
|
|
208
|
+
"""
|
|
209
|
+
from . import _env
|
|
210
|
+
|
|
211
|
+
raw, var = _env.lookup("VOLUMES")
|
|
212
|
+
if not raw:
|
|
213
|
+
return []
|
|
214
|
+
try:
|
|
215
|
+
data = json.loads(raw)
|
|
216
|
+
except ValueError:
|
|
217
|
+
import warnings
|
|
218
|
+
warnings.warn(
|
|
219
|
+
f"{var} is set but contains invalid JSON; no volumes will be "
|
|
220
|
+
"resolved. This is a platform configuration bug.",
|
|
221
|
+
RuntimeWarning,
|
|
222
|
+
stacklevel=2,
|
|
223
|
+
)
|
|
224
|
+
return []
|
|
225
|
+
return data if isinstance(data, list) else []
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _resolve_mount(name: str) -> Optional[Dict[str, Any]]:
|
|
229
|
+
for m in _mounted_volumes():
|
|
230
|
+
if isinstance(m, dict) and m.get("name") == name:
|
|
231
|
+
return m
|
|
232
|
+
return None
|
bladekit/_webhook.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Completion-webhook config + receiver-side verification (async-jobs-study.md).
|
|
2
|
+
|
|
3
|
+
A deployed endpoint job spawned with ``webhook=`` gets a POST to your URL when
|
|
4
|
+
it finishes. The control plane signs the body Stripe-style with
|
|
5
|
+
``Forge-Signature: t=<unix>,v1=<hex hmac-sha256(secret, "<t>.<raw_body>")>`` —
|
|
6
|
+
``verify_webhook`` checks it and returns the parsed payload.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import hmac
|
|
13
|
+
import json
|
|
14
|
+
import time
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from typing import Any, Mapping, Optional, Tuple
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class Webhook:
|
|
21
|
+
"""Completion webhook for ``spawn(webhook=...)``.
|
|
22
|
+
|
|
23
|
+
``secret`` is the HMAC signing secret; leave it None and the control plane
|
|
24
|
+
generates one, returned ONCE on the spawn response
|
|
25
|
+
(``DeployedFunctionCall.webhook_secret``). ``signed=False`` opts out of
|
|
26
|
+
signing (no FORGE_SECRET_KEY needed on the control plane).
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
url: str
|
|
30
|
+
secret: Optional[str] = None
|
|
31
|
+
signed: bool = True
|
|
32
|
+
events: Tuple[str, ...] = field(default_factory=lambda: ("completed",))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _webhook_dict(w: "Webhook | dict | str | None") -> Optional[dict]:
|
|
36
|
+
"""Normalize a webhook spec (Webhook | dict | bare-URL str) to the wire dict."""
|
|
37
|
+
if w is None:
|
|
38
|
+
return None
|
|
39
|
+
if isinstance(w, dict):
|
|
40
|
+
return w
|
|
41
|
+
if isinstance(w, str):
|
|
42
|
+
return {"url": w}
|
|
43
|
+
if isinstance(w, Webhook):
|
|
44
|
+
d: dict[str, Any] = {"url": w.url, "signed": w.signed, "events": list(w.events)}
|
|
45
|
+
if w.secret:
|
|
46
|
+
d["secret"] = w.secret
|
|
47
|
+
return d
|
|
48
|
+
raise TypeError(f"webhook must be a bladekit.Webhook, dict, url str or None, got {type(w)!r}")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _header(headers: Mapping[str, Any], name: str) -> Optional[str]:
|
|
52
|
+
lname = name.lower()
|
|
53
|
+
for k, v in headers.items():
|
|
54
|
+
if k.lower() == lname:
|
|
55
|
+
# Header values may be a list (Forge web_endpoint item.headers gives
|
|
56
|
+
# {name: [values]}, mirroring Go's http.Header) or a plain string
|
|
57
|
+
# (WSGI/ASGI request.headers). Normalize to the first string value.
|
|
58
|
+
if isinstance(v, (list, tuple)):
|
|
59
|
+
return str(v[0]) if v else None
|
|
60
|
+
return str(v)
|
|
61
|
+
return None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def verify_webhook(
|
|
65
|
+
secret: str,
|
|
66
|
+
headers: Mapping[str, str],
|
|
67
|
+
raw_body: "bytes | str",
|
|
68
|
+
*,
|
|
69
|
+
tolerance: int = 300,
|
|
70
|
+
) -> dict:
|
|
71
|
+
"""Verify an AI Fabric completion-webhook and return its parsed JSON payload.
|
|
72
|
+
|
|
73
|
+
Raises ValueError on a missing/malformed/stale/forged signature. ``raw_body``
|
|
74
|
+
MUST be the exact bytes received (do not re-serialize). ``tolerance`` is the
|
|
75
|
+
max clock skew in seconds (0 disables the timestamp check).
|
|
76
|
+
"""
|
|
77
|
+
sig = _header(headers, "Forge-Signature")
|
|
78
|
+
if not sig:
|
|
79
|
+
raise ValueError("missing Forge-Signature header")
|
|
80
|
+
parts = dict(p.split("=", 1) for p in sig.split(",") if "=" in p)
|
|
81
|
+
ts, v1 = parts.get("t"), parts.get("v1")
|
|
82
|
+
if not ts or not v1:
|
|
83
|
+
raise ValueError("malformed Forge-Signature header")
|
|
84
|
+
if tolerance and abs(time.time() - int(ts)) > tolerance:
|
|
85
|
+
raise ValueError("webhook timestamp outside tolerance (possible replay)")
|
|
86
|
+
body = raw_body if isinstance(raw_body, (bytes, bytearray)) else raw_body.encode()
|
|
87
|
+
mac = hmac.new(secret.encode(), (ts + ".").encode() + body, hashlib.sha256).hexdigest()
|
|
88
|
+
if not hmac.compare_digest(mac, v1):
|
|
89
|
+
raise ValueError("webhook signature mismatch")
|
|
90
|
+
return json.loads(body)
|