bladekit 4.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bladekit/__init__.py +219 -0
- bladekit/_app.py +3241 -0
- bladekit/_banner.py +640 -0
- bladekit/_batch.py +150 -0
- bladekit/_bucket.py +211 -0
- bladekit/_build.py +337 -0
- bladekit/_bundle.py +459 -0
- bladekit/_client.py +1152 -0
- bladekit/_cls.py +574 -0
- bladekit/_constants.py +121 -0
- bladekit/_cron.py +88 -0
- bladekit/_dict.py +174 -0
- bladekit/_env.py +98 -0
- bladekit/_image.py +577 -0
- bladekit/_loader.py +79 -0
- bladekit/_map.py +555 -0
- bladekit/_preempt.py +94 -0
- bladekit/_queue.py +137 -0
- bladekit/_render.py +683 -0
- bladekit/_runner_api.py +148 -0
- bladekit/_sandbox.py +1051 -0
- bladekit/_secret.py +52 -0
- bladekit/_serialize.py +88 -0
- bladekit/_serve.py +188 -0
- bladekit/_volume.py +232 -0
- bladekit/_webhook.py +90 -0
- bladekit/cli.py +2282 -0
- bladekit/config.py +181 -0
- bladekit/exceptions.py +544 -0
- bladekit-4.5.0.dist-info/METADATA +373 -0
- bladekit-4.5.0.dist-info/RECORD +33 -0
- bladekit-4.5.0.dist-info/WHEEL +4 -0
- bladekit-4.5.0.dist-info/entry_points.txt +3 -0
bladekit/_app.py
ADDED
|
@@ -0,0 +1,3241 @@
|
|
|
1
|
+
"""Object API: ``App``, ``Function``, ``FunctionCall`` (spawn handle).
|
|
2
|
+
|
|
3
|
+
This is the ergonomic surface (SPEC §1). It also owns the run lifecycle a single
|
|
4
|
+
``blade run`` (or interactive session) maps onto:
|
|
5
|
+
|
|
6
|
+
- The *first* remote invocation of a function creates a code bundle, a run, and
|
|
7
|
+
the first call. Subsequent ``.remote()`` / ``.spawn()`` of the *same* function
|
|
8
|
+
in the same process reuse the same run — i.e. the same hot container — by
|
|
9
|
+
appending calls (FIFO), per SPEC §3.3.
|
|
10
|
+
- ``complete_session()`` POSTs ``/complete`` on every open run so the container
|
|
11
|
+
drains immediately at script end instead of waiting for the idle timeout.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import sys
|
|
17
|
+
import time
|
|
18
|
+
import re
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, TYPE_CHECKING
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING: # pragma: no cover — typing only, avoids an import cycle
|
|
24
|
+
from ._map import MapHandle
|
|
25
|
+
|
|
26
|
+
from . import exceptions as exc
|
|
27
|
+
from ._constants import (
|
|
28
|
+
CALL_TERMINAL_STATES,
|
|
29
|
+
DEFAULT_CPU_MILLICORES,
|
|
30
|
+
DEFAULT_IDLE_TIMEOUT_SECONDS,
|
|
31
|
+
DEFAULT_MEMORY_MB,
|
|
32
|
+
DEFAULT_TIMEOUT_SECONDS,
|
|
33
|
+
GPU_COUNT_MAX,
|
|
34
|
+
GPU_COUNT_MIN,
|
|
35
|
+
POLL_INITIAL_INTERVAL,
|
|
36
|
+
POLL_MAX_INTERVAL,
|
|
37
|
+
PRIORITY_MAX,
|
|
38
|
+
PRIORITY_MIN,
|
|
39
|
+
RETRIES_BACKOFF_FACTOR_MAX,
|
|
40
|
+
RETRIES_BACKOFF_FACTOR_MIN,
|
|
41
|
+
RETRIES_BACKOFF_INITIAL_MAX,
|
|
42
|
+
RETRIES_BACKOFF_MAX_MAX,
|
|
43
|
+
RETRIES_MAX,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
_APP_NAME_RE = re.compile(r"^[a-z][a-z0-9-]{2,62}$")
|
|
47
|
+
|
|
48
|
+
# Parse "GPU_TYPE:N" for multi-GPU (conventions.md §6, scheduler-v2.md §5).
|
|
49
|
+
# Examples: "RTX-A4500" -> ("RTX-A4500", 1); "RTX-A4500:4" -> ("RTX-A4500", 4).
|
|
50
|
+
_GPU_COUNT_RE = re.compile(r"^(.+):(\d+)$")
|
|
51
|
+
_HTTP_METHODS = frozenset({"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"})
|
|
52
|
+
_ASGI_METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _python_version() -> str:
|
|
56
|
+
return f"{sys.version_info.major}.{sys.version_info.minor}"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class _Session:
|
|
61
|
+
"""Process-wide registry of open runs, so we can drain them at exit."""
|
|
62
|
+
|
|
63
|
+
client: Any = None
|
|
64
|
+
open_run_ids: List[str] = field(default_factory=list)
|
|
65
|
+
|
|
66
|
+
def register_run(self, run_id: str) -> None:
|
|
67
|
+
if run_id not in self.open_run_ids:
|
|
68
|
+
self.open_run_ids.append(run_id)
|
|
69
|
+
|
|
70
|
+
def complete_all(self) -> None:
|
|
71
|
+
if self.client is None:
|
|
72
|
+
return
|
|
73
|
+
for run_id in self.open_run_ids:
|
|
74
|
+
try:
|
|
75
|
+
self.client.complete_run(run_id)
|
|
76
|
+
except exc.FabricError:
|
|
77
|
+
pass # best-effort drain; idle timeout is the fallback
|
|
78
|
+
self.open_run_ids.clear()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# A module-level session keyed nowhere special: a `blade run` is one process,
|
|
82
|
+
# one script. The CLI calls `complete_session()` after the entrypoint returns.
|
|
83
|
+
_SESSION = _Session()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def complete_session() -> None:
|
|
87
|
+
"""POST /complete on every run opened in this process (drain hot containers)."""
|
|
88
|
+
_SESSION.complete_all()
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _emit_run_warnings(run: dict) -> None:
|
|
92
|
+
"""Print structured creation warnings in yellow (conventions.md §2).
|
|
93
|
+
|
|
94
|
+
GPU_TYPE_UNAVAILABLE lists the GPU types actually known to the fleet so the
|
|
95
|
+
user can fix the typo / pick an available type. Unknown warning codes are
|
|
96
|
+
shown verbatim (extensible registry — never branch logic on them).
|
|
97
|
+
"""
|
|
98
|
+
warnings = run.get("warnings")
|
|
99
|
+
if not warnings:
|
|
100
|
+
return
|
|
101
|
+
try:
|
|
102
|
+
from ._render import err_console
|
|
103
|
+
except Exception: # pragma: no cover - rich import failure is non-fatal
|
|
104
|
+
return
|
|
105
|
+
for w in warnings:
|
|
106
|
+
code = w.get("code", "WARNING")
|
|
107
|
+
message = w.get("message", "")
|
|
108
|
+
err_console.print(f"[yellow]warning[/yellow] [bold]{code}[/bold]: {message}")
|
|
109
|
+
if code == "GPU_TYPE_UNAVAILABLE":
|
|
110
|
+
details = w.get("details") or {}
|
|
111
|
+
known = details.get("known_types") or []
|
|
112
|
+
if known:
|
|
113
|
+
err_console.print(
|
|
114
|
+
f" [yellow]available GPU types:[/yellow] {', '.join(known)}"
|
|
115
|
+
)
|
|
116
|
+
else:
|
|
117
|
+
err_console.print(
|
|
118
|
+
" [yellow]no GPU types are currently online in the fleet.[/yellow]"
|
|
119
|
+
)
|
|
120
|
+
err_console.print(
|
|
121
|
+
" [dim]the run will stay pending until a node of this type "
|
|
122
|
+
"appears.[/dim]"
|
|
123
|
+
)
|
|
124
|
+
elif code == "ORG_GPU_QUOTA_REACHED":
|
|
125
|
+
details = w.get("details") or {}
|
|
126
|
+
in_use = details.get("in_use", "?")
|
|
127
|
+
limit = details.get("limit", "?")
|
|
128
|
+
gpu_type = details.get("gpu_type", "*")
|
|
129
|
+
scope = "GPUs" if gpu_type == "*" else f"GPUs of type {gpu_type!r}"
|
|
130
|
+
err_console.print(
|
|
131
|
+
f" [dim]your organization is using {in_use}/{limit} {scope}; "
|
|
132
|
+
"the run waits until usage drops below the limit.[/dim]"
|
|
133
|
+
)
|
|
134
|
+
if code in ("GPU_QUOTA_REACHED", "ORG_GPU_QUOTA_REACHED"):
|
|
135
|
+
# Let the run spinner say "waiting for quota" instead of the
|
|
136
|
+
# misleading "waiting for capacity" (_render.RunProgress).
|
|
137
|
+
try:
|
|
138
|
+
from ._render import mark_quota_queued
|
|
139
|
+
|
|
140
|
+
mark_quota_queued(run.get("run_id", ""))
|
|
141
|
+
except Exception: # pragma: no cover - render import is optional
|
|
142
|
+
pass
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass(frozen=True)
|
|
146
|
+
class Autoscale:
|
|
147
|
+
"""Per-endpoint autoscaling policy (autoscaling v2).
|
|
148
|
+
|
|
149
|
+
Every knob is optional and only ADDS to the default concurrency-based
|
|
150
|
+
autoscaler — guardrails (min/max clamp, cooldown, breaker) always win.
|
|
151
|
+
|
|
152
|
+
- ``target_inputs``: per-replica concurrency the autoscaler AIMS for;
|
|
153
|
+
``max_inputs`` (from ``@app.concurrent``) stays the admission ceiling
|
|
154
|
+
replicas may burst to while new capacity boots. Scale before saturation.
|
|
155
|
+
- ``buffer_replicas``: warm spare replicas kept only WHILE the endpoint is
|
|
156
|
+
active — an idle endpoint still settles to ``min_replicas``.
|
|
157
|
+
- ``vllm_target_waiting``: the curated vLLM preset (``@app.web_server``
|
|
158
|
+
endpoints only): scale on vLLM's scheduler queue so continuous batching
|
|
159
|
+
pressure is visible to the autoscaler. The platform owns the metrics
|
|
160
|
+
plumbing — no exporter config, no PromQL.
|
|
161
|
+
- ``scaledown_window``: stabilization window (seconds, 30..3600) of the
|
|
162
|
+
PROPORTIONAL scale-down path — how long the endpoint must be
|
|
163
|
+
continuously observed needing fewer replicas before a surplus replica
|
|
164
|
+
is shed. Unset = ``max(server default, scale_down_idle_seconds)``. It
|
|
165
|
+
never affects scale-up, the hard ``max_replicas`` cap, or
|
|
166
|
+
scale-to-zero (which stays governed by ``scale_down_idle_seconds``).
|
|
167
|
+
Requires a control plane of 2026-08-11 or newer (contracts CHANGELOG,
|
|
168
|
+
autoscaling anti-flap entry): older control planes reject the unknown
|
|
169
|
+
policy field with ``400 SCALING_POLICY_INVALID``.
|
|
170
|
+
|
|
171
|
+
Example::
|
|
172
|
+
|
|
173
|
+
@app.web_server(
|
|
174
|
+
startup_timeout=600, min_replicas=1, max_replicas=6,
|
|
175
|
+
autoscale=bladekit.Autoscale(target_inputs=24, buffer_replicas=1,
|
|
176
|
+
vllm_target_waiting=4),
|
|
177
|
+
)
|
|
178
|
+
@app.concurrent(max_inputs=32)
|
|
179
|
+
@app.function(gpu="RTX-A4500", image=vllm_image)
|
|
180
|
+
def serve(): ...
|
|
181
|
+
"""
|
|
182
|
+
|
|
183
|
+
target_inputs: Optional[int] = None
|
|
184
|
+
buffer_replicas: Optional[int] = None
|
|
185
|
+
vllm_target_waiting: Optional[int] = None
|
|
186
|
+
scaledown_window: Optional[int] = None
|
|
187
|
+
|
|
188
|
+
def __post_init__(self) -> None:
|
|
189
|
+
if (
|
|
190
|
+
self.target_inputs is None
|
|
191
|
+
and self.buffer_replicas is None
|
|
192
|
+
and self.vllm_target_waiting is None
|
|
193
|
+
and self.scaledown_window is None
|
|
194
|
+
):
|
|
195
|
+
raise exc.InvalidArgumentError(
|
|
196
|
+
"SCALING_POLICY_INVALID",
|
|
197
|
+
"bladekit.Autoscale requires at least one of target_inputs, "
|
|
198
|
+
"buffer_replicas, vllm_target_waiting, scaledown_window",
|
|
199
|
+
)
|
|
200
|
+
if self.target_inputs is not None and (
|
|
201
|
+
not isinstance(self.target_inputs, int)
|
|
202
|
+
or isinstance(self.target_inputs, bool)
|
|
203
|
+
or self.target_inputs < 1
|
|
204
|
+
or self.target_inputs > 100
|
|
205
|
+
):
|
|
206
|
+
raise exc.InvalidArgumentError(
|
|
207
|
+
"SCALING_POLICY_INVALID",
|
|
208
|
+
"target_inputs must be an integer between 1 and 100",
|
|
209
|
+
)
|
|
210
|
+
if self.buffer_replicas is not None and (
|
|
211
|
+
not isinstance(self.buffer_replicas, int)
|
|
212
|
+
or isinstance(self.buffer_replicas, bool)
|
|
213
|
+
or self.buffer_replicas < 0
|
|
214
|
+
or self.buffer_replicas > 10
|
|
215
|
+
):
|
|
216
|
+
raise exc.InvalidArgumentError(
|
|
217
|
+
"SCALING_POLICY_INVALID",
|
|
218
|
+
"buffer_replicas must be an integer between 0 and 10",
|
|
219
|
+
)
|
|
220
|
+
if self.vllm_target_waiting is not None and (
|
|
221
|
+
not isinstance(self.vllm_target_waiting, int)
|
|
222
|
+
or isinstance(self.vllm_target_waiting, bool)
|
|
223
|
+
or self.vllm_target_waiting < 1
|
|
224
|
+
or self.vllm_target_waiting > 1000
|
|
225
|
+
):
|
|
226
|
+
raise exc.InvalidArgumentError(
|
|
227
|
+
"SCALING_POLICY_INVALID",
|
|
228
|
+
"vllm_target_waiting must be an integer between 1 and 1000",
|
|
229
|
+
)
|
|
230
|
+
# Mirrors the server rule (scaledown_window_seconds 30..3600, the same
|
|
231
|
+
# bounds as scale_down_idle_seconds).
|
|
232
|
+
if self.scaledown_window is not None and (
|
|
233
|
+
not isinstance(self.scaledown_window, int)
|
|
234
|
+
or isinstance(self.scaledown_window, bool)
|
|
235
|
+
or self.scaledown_window < 30
|
|
236
|
+
or self.scaledown_window > 3600
|
|
237
|
+
):
|
|
238
|
+
raise exc.InvalidArgumentError(
|
|
239
|
+
"SCALING_POLICY_INVALID",
|
|
240
|
+
"scaledown_window must be an integer between 30 and 3600",
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
@dataclass
|
|
245
|
+
class EndpointDeclaration:
|
|
246
|
+
"""SDK-side endpoint metadata attached by ``@app.web_endpoint`` / ASGI."""
|
|
247
|
+
|
|
248
|
+
name: Optional[str]
|
|
249
|
+
kind: str
|
|
250
|
+
path: str
|
|
251
|
+
methods: tuple[str, ...]
|
|
252
|
+
min_replicas: Optional[int] = None
|
|
253
|
+
max_replicas: Optional[int] = None
|
|
254
|
+
max_concurrent_inputs: Optional[int] = None
|
|
255
|
+
scale_down_idle_seconds: Optional[int] = None
|
|
256
|
+
# Ingress queue bound (0 = unbounded); sent only when set — the server
|
|
257
|
+
# default (0) preserves the historical unbounded queueing.
|
|
258
|
+
max_queue_depth: Optional[int] = None
|
|
259
|
+
reserved_replicas: Optional[int] = None # hybrid: always-on floor on 'reserved'
|
|
260
|
+
public: bool = False # public endpoint (no auth); private by default
|
|
261
|
+
# §13.5 — local HTTP server endpoints (web_server / asgi_app).
|
|
262
|
+
port: Optional[int] = None # server_port: user-provided for web_server, None for asgi_app
|
|
263
|
+
startup_timeout_seconds: Optional[int] = None # loopback readiness budget
|
|
264
|
+
# Autoscaling v2 — policy declared on the endpoint decorator (autoscale=).
|
|
265
|
+
autoscale: Optional[Autoscale] = None
|
|
266
|
+
# GPU snapshot opt-in (coldstart-snapshot.md §5). Valid ONLY on web_server
|
|
267
|
+
# with gpu_count==1 — validated in _endpoint_specs() (mirror of the server).
|
|
268
|
+
snapshot: bool = False
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
@dataclass
|
|
272
|
+
class DeploymentManifest:
|
|
273
|
+
"""Serializable deployment manifest derived from an ``App``.
|
|
274
|
+
|
|
275
|
+
``to_dict()`` is intentionally richer than ``DeploymentCreateRequest`` so
|
|
276
|
+
local tooling can inspect functions/classes/resources/images/secrets/volumes.
|
|
277
|
+
``to_deployment_request()`` narrows it to the current control-plane contract.
|
|
278
|
+
"""
|
|
279
|
+
|
|
280
|
+
app_name: str
|
|
281
|
+
bundle_id: Optional[str]
|
|
282
|
+
runtime: dict
|
|
283
|
+
endpoints: list[dict]
|
|
284
|
+
image: Optional[str] = None
|
|
285
|
+
image_build_id: Optional[str] = None
|
|
286
|
+
labels: Optional[dict[str, str]] = None
|
|
287
|
+
functions: list[dict] = field(default_factory=list)
|
|
288
|
+
classes: list[dict] = field(default_factory=list)
|
|
289
|
+
crons: list[dict] = field(default_factory=list)
|
|
290
|
+
|
|
291
|
+
def to_dict(self) -> dict:
|
|
292
|
+
data: dict[str, Any] = {
|
|
293
|
+
"app_name": self.app_name,
|
|
294
|
+
"runtime": self.runtime,
|
|
295
|
+
"endpoints": self.endpoints,
|
|
296
|
+
"functions": self.functions,
|
|
297
|
+
"classes": self.classes,
|
|
298
|
+
"crons": self.crons,
|
|
299
|
+
}
|
|
300
|
+
if self.bundle_id is not None:
|
|
301
|
+
data["bundle_id"] = self.bundle_id
|
|
302
|
+
if self.image is not None:
|
|
303
|
+
data["image"] = self.image
|
|
304
|
+
if self.image_build_id is not None:
|
|
305
|
+
data["image_build_id"] = self.image_build_id
|
|
306
|
+
if self.labels:
|
|
307
|
+
data["labels"] = self.labels
|
|
308
|
+
return data
|
|
309
|
+
|
|
310
|
+
def to_deployment_request(self) -> dict:
|
|
311
|
+
if not self.bundle_id:
|
|
312
|
+
raise exc.InvalidArgumentError(
|
|
313
|
+
"INVALID_ARGUMENT",
|
|
314
|
+
"deployment manifest has no bundle_id; build/upload the bundle first.",
|
|
315
|
+
)
|
|
316
|
+
if not self.endpoints and not self.crons:
|
|
317
|
+
raise exc.InvalidArgumentError(
|
|
318
|
+
"INVALID_ARGUMENT", "deployment manifest has no endpoints or crons."
|
|
319
|
+
)
|
|
320
|
+
if (self.image is None) == (self.image_build_id is None):
|
|
321
|
+
raise exc.InvalidArgumentError(
|
|
322
|
+
"INVALID_ARGUMENT",
|
|
323
|
+
"deployment manifest must have exactly one of image or image_build_id.",
|
|
324
|
+
)
|
|
325
|
+
body: dict[str, Any] = {
|
|
326
|
+
"bundle_id": self.bundle_id,
|
|
327
|
+
"runtime": self.runtime,
|
|
328
|
+
"endpoints": self.endpoints,
|
|
329
|
+
}
|
|
330
|
+
if self.crons:
|
|
331
|
+
body["crons"] = self.crons
|
|
332
|
+
if self.image is not None:
|
|
333
|
+
body["image"] = self.image
|
|
334
|
+
if self.image_build_id is not None:
|
|
335
|
+
body["image_build_id"] = self.image_build_id
|
|
336
|
+
if self.labels:
|
|
337
|
+
body["labels"] = self.labels
|
|
338
|
+
return body
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
class FunctionCall:
|
|
342
|
+
"""Handle returned by ``Function.spawn`` — non-blocking.
|
|
343
|
+
|
|
344
|
+
``.get(timeout=)`` blocks until the call finishes and returns the result (or
|
|
345
|
+
re-raises the remote exception); ``.cancel()`` cancels the whole run.
|
|
346
|
+
"""
|
|
347
|
+
|
|
348
|
+
def __init__(self, function: "Function", run_id: str, call_id: str) -> None:
|
|
349
|
+
self._function = function
|
|
350
|
+
self.run_id = run_id
|
|
351
|
+
self.call_id = call_id
|
|
352
|
+
|
|
353
|
+
def __repr__(self) -> str:
|
|
354
|
+
return f"FunctionCall(call_id={self.call_id!r}, run_id={self.run_id!r})"
|
|
355
|
+
|
|
356
|
+
def get(self, timeout: Optional[float] = None) -> Any:
|
|
357
|
+
return self._function._await_call(self.call_id, self.run_id, timeout=timeout)
|
|
358
|
+
|
|
359
|
+
def cancel(self) -> None:
|
|
360
|
+
"""Cancel the underlying run (a run == a container; cancel is run-level)."""
|
|
361
|
+
self._function._app._client_cancel(self.run_id)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _reject_coroutine_function(fn: Callable[..., Any], what: str = "an AI Fabric function") -> None:
|
|
365
|
+
"""Reject ``async def`` at decoration time instead of inside the container.
|
|
366
|
+
|
|
367
|
+
No runner path awaits a coroutine: a function call does ``fn(*args, **kwargs)``
|
|
368
|
+
(runner-protocol §3.4.d), class methods and their ``@bladekit.enter``/``build``/
|
|
369
|
+
``exit`` hooks are invoked the same way (§10), and both local-server kinds call
|
|
370
|
+
the decorated function synchronously — ``web_server`` to start the server,
|
|
371
|
+
``asgi_app`` to obtain the ASGI app object.
|
|
372
|
+
|
|
373
|
+
So an ``async def`` used to be accepted here, ship, get scheduled, pull its
|
|
374
|
+
image, and only then fail deep in the container with "cannot pickle 'coroutine'
|
|
375
|
+
object" (or, on a web endpoint, "unsupported HTTP response body type") — an
|
|
376
|
+
error naming neither the cause nor the fix. A lifecycle hook was worse than an
|
|
377
|
+
error: the coroutine was never awaited, so ``@bladekit.enter`` silently did not
|
|
378
|
+
load the model and the failure surfaced later as an unrelated AttributeError.
|
|
379
|
+
|
|
380
|
+
``asgi_app``/``web_server`` are not exceptions: their factory is sync and
|
|
381
|
+
returns the async application, which is where user coroutines legitimately
|
|
382
|
+
live. Awaiting user coroutines in the runner is a separate change; until it
|
|
383
|
+
lands, failing at the decorator is the honest behaviour.
|
|
384
|
+
"""
|
|
385
|
+
import inspect # noqa: PLC0415 - only needed on the decoration path
|
|
386
|
+
|
|
387
|
+
if not inspect.iscoroutinefunction(fn):
|
|
388
|
+
return
|
|
389
|
+
name = getattr(fn, "__name__", "<function>")
|
|
390
|
+
raise exc.InvalidArgumentError(
|
|
391
|
+
"INVALID_ARGUMENT",
|
|
392
|
+
f"{name!r} is an 'async def' function, which AI Fabric cannot execute yet as "
|
|
393
|
+
f"{what}: the runner calls it synchronously. Wrap the coroutine, e.g.\n"
|
|
394
|
+
f" def {name}(*args, **kwargs):\n"
|
|
395
|
+
f" return asyncio.run(_{name}(*args, **kwargs))\n"
|
|
396
|
+
"For an ASGI app (FastAPI, Starlette), use @app.asgi_app on a SYNC "
|
|
397
|
+
"factory that returns the application object.",
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
class Function:
|
|
402
|
+
"""A decorated function, callable remotely."""
|
|
403
|
+
|
|
404
|
+
def __init__(
|
|
405
|
+
self,
|
|
406
|
+
app: "App",
|
|
407
|
+
fn: Callable[..., Any],
|
|
408
|
+
*,
|
|
409
|
+
gpu: Optional[str] = None,
|
|
410
|
+
cpu: Optional[float] = None,
|
|
411
|
+
memory: Optional[int] = None,
|
|
412
|
+
timeout: Optional[int] = None,
|
|
413
|
+
image: Any = None,
|
|
414
|
+
idle_timeout: Optional[int] = None,
|
|
415
|
+
volumes: Optional[Dict[str, Any]] = None,
|
|
416
|
+
secrets: Optional[List[Any]] = None,
|
|
417
|
+
class_name: Optional[str] = None,
|
|
418
|
+
serve: bool = False,
|
|
419
|
+
retries: Optional[dict] = None,
|
|
420
|
+
priority: Optional[int] = None,
|
|
421
|
+
namespace: Optional[str] = None,
|
|
422
|
+
schedule: Any = None,
|
|
423
|
+
# spot/reserved capacity classes, preemption budget
|
|
424
|
+
gpu_class: str = "spot",
|
|
425
|
+
best_effort: bool = False,
|
|
426
|
+
max_interruptions: Optional[int] = None,
|
|
427
|
+
# Bound bladekit.parameter() values for a parametrized @app.cls method.
|
|
428
|
+
parameters: Optional[Dict[str, Any]] = None,
|
|
429
|
+
) -> None:
|
|
430
|
+
_validate_gpu_class(gpu_class)
|
|
431
|
+
_reject_coroutine_function(fn)
|
|
432
|
+
self._app = app
|
|
433
|
+
self._fn = fn
|
|
434
|
+
self.gpu = gpu
|
|
435
|
+
self.cpu = cpu
|
|
436
|
+
self.memory = memory
|
|
437
|
+
self.timeout = timeout
|
|
438
|
+
# ``image`` may be a string (alias/ref) or a ``bladekit.Image`` (DSL).
|
|
439
|
+
self.image = image
|
|
440
|
+
self.idle_timeout = idle_timeout
|
|
441
|
+
# {mount_path: bladekit.Volume}, [bladekit.Secret], @app.cls class name.
|
|
442
|
+
self.volumes = volumes
|
|
443
|
+
self.secrets = secrets
|
|
444
|
+
self.class_name = class_name
|
|
445
|
+
self.serve = serve
|
|
446
|
+
# retry policy, scheduling priority, namespace, cron schedule.
|
|
447
|
+
self.retries = _normalize_retries(retries)
|
|
448
|
+
self.priority = priority
|
|
449
|
+
self.namespace = namespace
|
|
450
|
+
self.schedule = schedule
|
|
451
|
+
# capacity class + preemption budget (runner-protocol §15.2.1,
|
|
452
|
+
# m8-spot.md §13). gpu_class defaults to "spot" (the cheap, preemptible
|
|
453
|
+
# default); best_effort and max_interruptions default to default behaviour.
|
|
454
|
+
self.gpu_class = gpu_class
|
|
455
|
+
self.best_effort = best_effort
|
|
456
|
+
self.max_interruptions = max_interruptions
|
|
457
|
+
# Bound bladekit.parameter() values (name -> value) for a parametrized
|
|
458
|
+
# @app.cls method; None for a plain function or an unparametrized class.
|
|
459
|
+
# Carried into FunctionRef.parameters so the runner instantiates the class
|
|
460
|
+
# with them and routes the call to the matching parametrized container.
|
|
461
|
+
self.parameters = parameters
|
|
462
|
+
self._endpoints: list[EndpointDeclaration] = []
|
|
463
|
+
self._max_concurrent_inputs: Optional[int] = None
|
|
464
|
+
# Autoscaling v2: function-level target concurrency from
|
|
465
|
+
# @app.concurrent(target_inputs=), folded into the endpoint's
|
|
466
|
+
# scaling_policy at manifest time (_endpoint_specs).
|
|
467
|
+
self._target_inputs: Optional[int] = None
|
|
468
|
+
# The run reused for hot-container call batching in this process.
|
|
469
|
+
self._run_id: Optional[str] = None
|
|
470
|
+
# Make the wrapped function still usable locally / introspectable.
|
|
471
|
+
self.__name__ = getattr(fn, "__name__", "fn")
|
|
472
|
+
self.__qualname__ = getattr(fn, "__qualname__", self.__name__)
|
|
473
|
+
self.__doc__ = getattr(fn, "__doc__", None)
|
|
474
|
+
|
|
475
|
+
for decl in getattr(fn, "__forge_endpoints__", []):
|
|
476
|
+
self._add_endpoint(decl)
|
|
477
|
+
pending_concurrency = getattr(fn, "__forge_max_concurrent_inputs__", None)
|
|
478
|
+
if pending_concurrency is not None:
|
|
479
|
+
self._set_concurrency(pending_concurrency)
|
|
480
|
+
pending_target = getattr(fn, "__forge_target_inputs__", None)
|
|
481
|
+
if pending_target is not None:
|
|
482
|
+
self._set_target_inputs(pending_target)
|
|
483
|
+
# Dynamic batching (@bladekit.batched): the wrapped function carries its
|
|
484
|
+
# batch config, which rides on the run/deploy manifest so the runner
|
|
485
|
+
# accumulates individual calls into batches (batching.md §4). None for a
|
|
486
|
+
# plain function — the wire body is then byte-for-byte unchanged.
|
|
487
|
+
self.batch_config: Optional[dict] = getattr(fn, "__forge_batched__", None)
|
|
488
|
+
|
|
489
|
+
def __repr__(self) -> str:
|
|
490
|
+
return f"<bladekit.Function {self._app.name}.{self.__name__} gpu={self.gpu!r}>"
|
|
491
|
+
|
|
492
|
+
@classmethod
|
|
493
|
+
def from_name(
|
|
494
|
+
cls, app_name: str, function_name: str, *, version: Optional[int] = None
|
|
495
|
+
) -> "_NamedFunction":
|
|
496
|
+
"""Reference a deployed function by name.
|
|
497
|
+
|
|
498
|
+
The public contract does not yet define SDK invocation of deployed
|
|
499
|
+
functions by name. The lightweight object is useful for discovery and
|
|
500
|
+
fails clearly if called.
|
|
501
|
+
|
|
502
|
+
``version``: pin to a deployment version. The endpoint is resolved once
|
|
503
|
+
and cached, so without a pin a long-lived reference silently follows a
|
|
504
|
+
redeploy or rollback. With one, a version change raises
|
|
505
|
+
``NotFoundError("DEPLOYMENT_VERSION_MISMATCH")`` instead — the call fails
|
|
506
|
+
where it can be handled rather than reaching different code than intended.
|
|
507
|
+
This pins RESOLUTION, not execution: a superseded deployment has no
|
|
508
|
+
replicas, so it cannot be invoked.
|
|
509
|
+
"""
|
|
510
|
+
return _NamedFunction(app_name, function_name, version=version)
|
|
511
|
+
|
|
512
|
+
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
|
513
|
+
"""Direct invocation runs the wrapped function in-process (no fleet).
|
|
514
|
+
|
|
515
|
+
This is load-bearing for remote execution: `FunctionRef.qualname`
|
|
516
|
+
points at the *decorated* object, so when forge-runner imports the
|
|
517
|
+
bundle module and calls `fn(*args, **kwargs)` (runner-protocol §3.4.d)
|
|
518
|
+
it is THIS method that executes the user code inside the container.
|
|
519
|
+
Locally it makes the decorated function behave like the plain one.
|
|
520
|
+
"""
|
|
521
|
+
return self._fn(*args, **kwargs)
|
|
522
|
+
|
|
523
|
+
def local(self, *args: Any, **kwargs: Any) -> Any:
|
|
524
|
+
"""Call the function locally (no remote execution). Handy for tests."""
|
|
525
|
+
return self._fn(*args, **kwargs)
|
|
526
|
+
|
|
527
|
+
def _add_endpoint(self, decl: EndpointDeclaration) -> None:
|
|
528
|
+
name = decl.name or self.__name__
|
|
529
|
+
normalized = EndpointDeclaration(
|
|
530
|
+
name=name,
|
|
531
|
+
kind=decl.kind,
|
|
532
|
+
path=decl.path,
|
|
533
|
+
methods=decl.methods,
|
|
534
|
+
min_replicas=decl.min_replicas,
|
|
535
|
+
max_replicas=decl.max_replicas,
|
|
536
|
+
max_concurrent_inputs=(
|
|
537
|
+
decl.max_concurrent_inputs or self._max_concurrent_inputs
|
|
538
|
+
),
|
|
539
|
+
scale_down_idle_seconds=decl.scale_down_idle_seconds,
|
|
540
|
+
max_queue_depth=decl.max_queue_depth,
|
|
541
|
+
reserved_replicas=decl.reserved_replicas,
|
|
542
|
+
public=decl.public,
|
|
543
|
+
port=decl.port,
|
|
544
|
+
startup_timeout_seconds=decl.startup_timeout_seconds,
|
|
545
|
+
autoscale=decl.autoscale,
|
|
546
|
+
snapshot=decl.snapshot,
|
|
547
|
+
)
|
|
548
|
+
self._endpoints.append(normalized)
|
|
549
|
+
|
|
550
|
+
def _set_concurrency(self, max_concurrent_inputs: int) -> None:
|
|
551
|
+
_validate_concurrency(max_concurrent_inputs)
|
|
552
|
+
self._max_concurrent_inputs = max_concurrent_inputs
|
|
553
|
+
for i, decl in enumerate(self._endpoints):
|
|
554
|
+
self._endpoints[i] = EndpointDeclaration(
|
|
555
|
+
name=decl.name,
|
|
556
|
+
kind=decl.kind,
|
|
557
|
+
path=decl.path,
|
|
558
|
+
methods=decl.methods,
|
|
559
|
+
min_replicas=decl.min_replicas,
|
|
560
|
+
max_replicas=decl.max_replicas,
|
|
561
|
+
max_concurrent_inputs=max_concurrent_inputs,
|
|
562
|
+
scale_down_idle_seconds=decl.scale_down_idle_seconds,
|
|
563
|
+
max_queue_depth=decl.max_queue_depth,
|
|
564
|
+
reserved_replicas=decl.reserved_replicas,
|
|
565
|
+
public=decl.public,
|
|
566
|
+
port=decl.port,
|
|
567
|
+
startup_timeout_seconds=decl.startup_timeout_seconds,
|
|
568
|
+
autoscale=decl.autoscale,
|
|
569
|
+
snapshot=decl.snapshot,
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
def _set_target_inputs(self, target_inputs: int) -> None:
|
|
573
|
+
if (
|
|
574
|
+
not isinstance(target_inputs, int)
|
|
575
|
+
or isinstance(target_inputs, bool)
|
|
576
|
+
or target_inputs < 1
|
|
577
|
+
or target_inputs > 100
|
|
578
|
+
):
|
|
579
|
+
raise exc.InvalidArgumentError(
|
|
580
|
+
"SCALING_POLICY_INVALID",
|
|
581
|
+
"target_inputs must be an integer between 1 and 100",
|
|
582
|
+
)
|
|
583
|
+
self._target_inputs = target_inputs
|
|
584
|
+
|
|
585
|
+
# -- resource / handshake builders -------------------------------------
|
|
586
|
+
|
|
587
|
+
def _parse_gpu(self) -> tuple[str, int]:
|
|
588
|
+
"""Parse ``self.gpu`` into ``(gpu_type, gpu_count)``.
|
|
589
|
+
|
|
590
|
+
Supports ``"RTX-A4500:4"`` syntax (scheduler-v2.md §5).
|
|
591
|
+
Validates gpu_count in [0, GPU_COUNT_MAX].
|
|
592
|
+
"""
|
|
593
|
+
raw = self.gpu or "none"
|
|
594
|
+
if raw == "none":
|
|
595
|
+
return "none", 0
|
|
596
|
+
m = _GPU_COUNT_RE.match(raw)
|
|
597
|
+
if m:
|
|
598
|
+
gpu_type = m.group(1)
|
|
599
|
+
gpu_count = int(m.group(2))
|
|
600
|
+
else:
|
|
601
|
+
gpu_type = raw
|
|
602
|
+
gpu_count = 1
|
|
603
|
+
# Validate gpu_count (conventions.md §6: 0–8).
|
|
604
|
+
if gpu_count < GPU_COUNT_MIN or gpu_count > GPU_COUNT_MAX:
|
|
605
|
+
raise exc.InvalidArgumentError(
|
|
606
|
+
"INVALID_ARGUMENT",
|
|
607
|
+
f"gpu_count={gpu_count} is out of range [{GPU_COUNT_MIN}, {GPU_COUNT_MAX}]; "
|
|
608
|
+
f"use gpu=\"{gpu_type}:N\" with N between 1 and {GPU_COUNT_MAX}.",
|
|
609
|
+
)
|
|
610
|
+
return gpu_type, gpu_count
|
|
611
|
+
|
|
612
|
+
def _resource_spec(self) -> dict:
|
|
613
|
+
gpu_type, gpu_count = self._parse_gpu()
|
|
614
|
+
spec: dict[str, Any] = {"gpu_type": gpu_type, "gpu_count": gpu_count}
|
|
615
|
+
if self.cpu is not None:
|
|
616
|
+
# round() not int(): cpu=1.001 should map to 1001 millicores, not 1000
|
|
617
|
+
# (int truncates). Keeps the millicore value faithful to the request.
|
|
618
|
+
millicores = round(self.cpu * 1000)
|
|
619
|
+
# Server floor is 100 millicores (openapi ResourceSpec.cpu_millicores,
|
|
620
|
+
# conventions §6). Catch cpu<0.1 locally with a clear message instead of
|
|
621
|
+
# a late, opaque 400 INVALID_ARGUMENT from the server.
|
|
622
|
+
if millicores < 100:
|
|
623
|
+
raise exc.InvalidArgumentError(
|
|
624
|
+
"INVALID_ARGUMENT",
|
|
625
|
+
f"cpu={self.cpu!r} is below the minimum of 0.1 cores "
|
|
626
|
+
f"(100 millicores); raise cpu= on @app.function.",
|
|
627
|
+
)
|
|
628
|
+
spec["cpu_millicores"] = millicores
|
|
629
|
+
else:
|
|
630
|
+
spec["cpu_millicores"] = DEFAULT_CPU_MILLICORES
|
|
631
|
+
spec["memory_mb"] = self.memory if self.memory is not None else DEFAULT_MEMORY_MB
|
|
632
|
+
spec["timeout_seconds"] = (
|
|
633
|
+
self.timeout if self.timeout is not None else DEFAULT_TIMEOUT_SECONDS
|
|
634
|
+
)
|
|
635
|
+
spec["idle_timeout_seconds"] = (
|
|
636
|
+
self.idle_timeout
|
|
637
|
+
if self.idle_timeout is not None
|
|
638
|
+
else DEFAULT_IDLE_TIMEOUT_SECONDS
|
|
639
|
+
)
|
|
640
|
+
return spec
|
|
641
|
+
|
|
642
|
+
def _function_ref(self) -> dict:
|
|
643
|
+
module = getattr(self._fn, "__module__", "__main__")
|
|
644
|
+
# The bundle root is the entrypoint script's directory; a function in the
|
|
645
|
+
# entrypoint module is importable as that script's module name. The CLI
|
|
646
|
+
# sets the module explicitly when running a file; in-process callers fall
|
|
647
|
+
# back to __module__.
|
|
648
|
+
module = self._app._module_for(module)
|
|
649
|
+
ref: dict[str, Any] = {"module": module, "qualname": self.__qualname__}
|
|
650
|
+
# @app.cls: class_name discriminates "instantiate then call method"
|
|
651
|
+
# from a plain function (runner-protocol.md §10). Absent for plain functions.
|
|
652
|
+
if self.class_name:
|
|
653
|
+
ref["class_name"] = self.class_name
|
|
654
|
+
# Parametrized @app.cls (bladekit.parameter): the bound values travel in the
|
|
655
|
+
# ref so the agent injects FORGE_FUNCTION_PARAMETERS and the runner
|
|
656
|
+
# instantiates the class with them (routing by parameter value). Absent
|
|
657
|
+
# (omitted) for a plain function or an unparametrized class.
|
|
658
|
+
if self.parameters:
|
|
659
|
+
ref["parameters"] = dict(self.parameters)
|
|
660
|
+
return ref
|
|
661
|
+
|
|
662
|
+
def _resolve_image(self) -> tuple[Optional[str], Optional[str]]:
|
|
663
|
+
"""Resolve the image into ``(image_str, image_build_id)`` for the run body.
|
|
664
|
+
|
|
665
|
+
- ``bladekit.Image`` (DSL): ensure the build (idempotent), stream build
|
|
666
|
+
logs if not yet succeeded, then return its build id.
|
|
667
|
+
- string (alias / registry ref): returned as-is, build id None — except a
|
|
668
|
+
``fabric/<name>`` alias, sent as ``forge/<name>`` (``to_wire_image``).
|
|
669
|
+
"""
|
|
670
|
+
from ._image import Image
|
|
671
|
+
|
|
672
|
+
img = self.image if self.image is not None else self._app.default_image
|
|
673
|
+
if img is None:
|
|
674
|
+
raise exc.FabricError(
|
|
675
|
+
f"function {self.__name__!r} has no image: pass image=... to "
|
|
676
|
+
"@app.function or set a default image on the App."
|
|
677
|
+
)
|
|
678
|
+
if isinstance(img, Image):
|
|
679
|
+
build_id = self._app._ensure_image_build(img)
|
|
680
|
+
return None, build_id
|
|
681
|
+
from ._image import to_wire_image
|
|
682
|
+
|
|
683
|
+
return to_wire_image(img), None
|
|
684
|
+
|
|
685
|
+
# -- run lifecycle ------------------------------------------------------
|
|
686
|
+
|
|
687
|
+
def _ensure_run(self) -> str:
|
|
688
|
+
"""Create (once) the run backing this function's calls in this process."""
|
|
689
|
+
if self._run_id is not None:
|
|
690
|
+
return self._run_id
|
|
691
|
+
|
|
692
|
+
client = self._app._require_client()
|
|
693
|
+
bundle_id = self._app._ensure_bundle()
|
|
694
|
+
|
|
695
|
+
from ._serialize import cloudpickle_version
|
|
696
|
+
|
|
697
|
+
# Resolve image first: a DSL image triggers (and waits on) a build,
|
|
698
|
+
# streaming build logs to the terminal. Either `image` or
|
|
699
|
+
# `image_build_id` is set, never both (RunCreateRequest, openapi).
|
|
700
|
+
image_str, image_build_id = self._resolve_image()
|
|
701
|
+
|
|
702
|
+
body: dict[str, Any] = {
|
|
703
|
+
"function": self._function_ref(),
|
|
704
|
+
"bundle_id": bundle_id,
|
|
705
|
+
"resources": self._resource_spec(),
|
|
706
|
+
"runtime": {
|
|
707
|
+
"python_version": _python_version(),
|
|
708
|
+
"cloudpickle_version": cloudpickle_version(),
|
|
709
|
+
},
|
|
710
|
+
}
|
|
711
|
+
if image_str is not None:
|
|
712
|
+
body["image"] = image_str
|
|
713
|
+
if image_build_id is not None:
|
|
714
|
+
body["image_build_id"] = image_build_id
|
|
715
|
+
if self.serve:
|
|
716
|
+
body["serve"] = True
|
|
717
|
+
volumes = self._volume_specs()
|
|
718
|
+
if volumes:
|
|
719
|
+
body["volumes"] = volumes
|
|
720
|
+
bucket_mounts = self._bucket_mount_specs()
|
|
721
|
+
if bucket_mounts:
|
|
722
|
+
body["bucket_mounts"] = bucket_mounts
|
|
723
|
+
secrets = self._secret_names()
|
|
724
|
+
if secrets:
|
|
725
|
+
body["secrets"] = secrets
|
|
726
|
+
if self._app.name:
|
|
727
|
+
body["app_name"] = self._app.name
|
|
728
|
+
# priority, namespace, retries (optional — absent = default behaviour).
|
|
729
|
+
if self.priority is not None:
|
|
730
|
+
_validate_priority(self.priority)
|
|
731
|
+
body["priority"] = self.priority
|
|
732
|
+
if self.namespace is not None:
|
|
733
|
+
body["namespace"] = self.namespace
|
|
734
|
+
if self.retries is not None:
|
|
735
|
+
body["retries"] = self.retries
|
|
736
|
+
# capacity class + preemption budget (m8-spot.md §13.6). Omitted
|
|
737
|
+
# when default so a default run body is byte-for-byte unchanged.
|
|
738
|
+
if self.gpu_class != "spot":
|
|
739
|
+
body["gpu_class"] = self.gpu_class
|
|
740
|
+
if self.best_effort:
|
|
741
|
+
body["best_effort"] = True
|
|
742
|
+
if self.max_interruptions is not None:
|
|
743
|
+
body["max_interruptions"] = self.max_interruptions
|
|
744
|
+
# dynamic batching config (batching.md §4). Omitted when the function is
|
|
745
|
+
# not batched so a default run body is byte-for-byte unchanged.
|
|
746
|
+
if self.batch_config is not None:
|
|
747
|
+
body["batch_config"] = self.batch_config
|
|
748
|
+
|
|
749
|
+
run = client.create_run(body)
|
|
750
|
+
self._run_id = run["run_id"]
|
|
751
|
+
_SESSION.client = client
|
|
752
|
+
_SESSION.register_run(self._run_id)
|
|
753
|
+
_emit_run_warnings(run)
|
|
754
|
+
return self._run_id
|
|
755
|
+
|
|
756
|
+
# -- volume / secret specs ------------------------------------------
|
|
757
|
+
|
|
758
|
+
def _volume_specs(self) -> list:
|
|
759
|
+
"""Build VolumeMountSpec list, create-or-get volumes that ask for it."""
|
|
760
|
+
if not self.volumes:
|
|
761
|
+
return []
|
|
762
|
+
from ._bucket import CloudBucketMount
|
|
763
|
+
from ._volume import Volume
|
|
764
|
+
|
|
765
|
+
client = self._app._require_client()
|
|
766
|
+
specs = []
|
|
767
|
+
for mount_path, vol in self.volumes.items():
|
|
768
|
+
# CloudBucketMount rides in the same map but serializes separately
|
|
769
|
+
# (see _bucket_mount_specs); skip it here.
|
|
770
|
+
if isinstance(vol, CloudBucketMount):
|
|
771
|
+
continue
|
|
772
|
+
if not isinstance(vol, Volume):
|
|
773
|
+
raise exc.InvalidArgumentError(
|
|
774
|
+
"INVALID_ARGUMENT",
|
|
775
|
+
f"volumes[{mount_path!r}] must be a bladekit.Volume or "
|
|
776
|
+
f"bladekit.CloudBucketMount, got {type(vol).__name__}",
|
|
777
|
+
)
|
|
778
|
+
if vol._create_if_missing:
|
|
779
|
+
client.create_volume(vol.name)
|
|
780
|
+
if vol._auto_commit_on_ready:
|
|
781
|
+
# Mirror of the server-side rule (runs.go): auto-commit fires
|
|
782
|
+
# on serving-ready, which one-shot runs never emit.
|
|
783
|
+
raise exc.InvalidArgumentError(
|
|
784
|
+
"INVALID_ARGUMENT",
|
|
785
|
+
f"volumes[{mount_path!r}]: auto_commit_on_ready is only "
|
|
786
|
+
"valid on deployed web_server/asgi_app endpoints, not on "
|
|
787
|
+
"one-shot runs",
|
|
788
|
+
)
|
|
789
|
+
if vol._auto_commit_every:
|
|
790
|
+
# Same rule, same reason: the periodic sweep is a replica
|
|
791
|
+
# feature, and a one-shot run commits with vol.commit().
|
|
792
|
+
raise exc.InvalidArgumentError(
|
|
793
|
+
"INVALID_ARGUMENT",
|
|
794
|
+
f"volumes[{mount_path!r}]: auto_commit_every is only valid "
|
|
795
|
+
"on deployed web_server/asgi_app endpoints, not on "
|
|
796
|
+
"one-shot runs",
|
|
797
|
+
)
|
|
798
|
+
spec: dict[str, Any] = {"name": vol.name, "mount_path": mount_path}
|
|
799
|
+
if vol._read_only:
|
|
800
|
+
spec["read_only"] = True
|
|
801
|
+
specs.append(spec)
|
|
802
|
+
return specs
|
|
803
|
+
|
|
804
|
+
def _secret_names(self) -> list:
|
|
805
|
+
if not self.secrets:
|
|
806
|
+
# No explicit secrets, but CloudBucketMount entries still need theirs.
|
|
807
|
+
return list(self._bucket_secret_names())
|
|
808
|
+
from ._secret import Secret
|
|
809
|
+
|
|
810
|
+
names = []
|
|
811
|
+
for s in self.secrets:
|
|
812
|
+
if not isinstance(s, Secret):
|
|
813
|
+
raise exc.InvalidArgumentError(
|
|
814
|
+
"INVALID_ARGUMENT",
|
|
815
|
+
f"secrets must be bladekit.Secret objects, got "
|
|
816
|
+
f"{type(s).__name__}",
|
|
817
|
+
)
|
|
818
|
+
names.append(s.name)
|
|
819
|
+
# Fold in the secrets that back any CloudBucketMount so their credentials
|
|
820
|
+
# are injected as env vars (the runner reads them at mount time).
|
|
821
|
+
for name in self._bucket_secret_names():
|
|
822
|
+
if name not in names:
|
|
823
|
+
names.append(name)
|
|
824
|
+
return names
|
|
825
|
+
|
|
826
|
+
def _secret_name_refs(self) -> list[str]:
|
|
827
|
+
# Explicit secrets plus the secrets that back any CloudBucketMount (their
|
|
828
|
+
# credentials must be injected as env vars for the runner to mount).
|
|
829
|
+
names = [getattr(s, "name", str(s)) for s in (self.secrets or [])]
|
|
830
|
+
for name in self._bucket_secret_names():
|
|
831
|
+
if name not in names:
|
|
832
|
+
names.append(name)
|
|
833
|
+
return names
|
|
834
|
+
|
|
835
|
+
def _bucket_secret_names(self) -> list[str]:
|
|
836
|
+
"""Secret names required by the function's CloudBucketMount entries."""
|
|
837
|
+
if not self.volumes:
|
|
838
|
+
return []
|
|
839
|
+
from ._bucket import CloudBucketMount
|
|
840
|
+
|
|
841
|
+
names: list[str] = []
|
|
842
|
+
for vol in self.volumes.values():
|
|
843
|
+
if isinstance(vol, CloudBucketMount) and vol.secret_name not in names:
|
|
844
|
+
names.append(vol.secret_name)
|
|
845
|
+
return names
|
|
846
|
+
|
|
847
|
+
def _bucket_mount_specs(self) -> list[dict]:
|
|
848
|
+
"""Build the BucketMountSpec list from CloudBucketMount entries.
|
|
849
|
+
|
|
850
|
+
Each spec carries the bucket name, mount path, options, and the *name* of
|
|
851
|
+
the secret that provides credentials — never a credential value. The
|
|
852
|
+
control plane resolves the secret and injects its values as container env
|
|
853
|
+
vars (mirroring ``secrets=``); the runner mounts the bucket at start.
|
|
854
|
+
"""
|
|
855
|
+
if not self.volumes:
|
|
856
|
+
return []
|
|
857
|
+
from ._bucket import CloudBucketMount
|
|
858
|
+
|
|
859
|
+
specs = []
|
|
860
|
+
for mount_path, vol in self.volumes.items():
|
|
861
|
+
if isinstance(vol, CloudBucketMount):
|
|
862
|
+
specs.append(vol._spec(mount_path))
|
|
863
|
+
return specs
|
|
864
|
+
|
|
865
|
+
def _volume_refs(self, *, include_auto_commit: bool = False) -> list[dict]:
|
|
866
|
+
"""Deploy-path volume refs. Like _volume_specs(), honours
|
|
867
|
+
create_if_missing so a deployed endpoint's volume is created (the RUN
|
|
868
|
+
path did this; the DEPLOY path silently dropped it).
|
|
869
|
+
|
|
870
|
+
``include_auto_commit``: only the ENDPOINT spec path carries the
|
|
871
|
+
auto-commit flag (it is validated against the endpoint kind there);
|
|
872
|
+
the function-manifest path omits it."""
|
|
873
|
+
if not self.volumes:
|
|
874
|
+
return []
|
|
875
|
+
from ._bucket import CloudBucketMount
|
|
876
|
+
from ._volume import Volume
|
|
877
|
+
|
|
878
|
+
client = self._app._require_client()
|
|
879
|
+
refs = []
|
|
880
|
+
for mount_path, vol in self.volumes.items():
|
|
881
|
+
# CloudBucketMount serializes via _bucket_mount_specs, not here.
|
|
882
|
+
if isinstance(vol, CloudBucketMount):
|
|
883
|
+
continue
|
|
884
|
+
if not isinstance(vol, Volume):
|
|
885
|
+
raise exc.InvalidArgumentError(
|
|
886
|
+
"INVALID_ARGUMENT",
|
|
887
|
+
f"volumes[{mount_path!r}] must be a bladekit.Volume or "
|
|
888
|
+
f"bladekit.CloudBucketMount, got {type(vol).__name__}",
|
|
889
|
+
)
|
|
890
|
+
if vol._create_if_missing:
|
|
891
|
+
client.create_volume(vol.name)
|
|
892
|
+
ref: dict[str, Any] = {"name": vol.name, "mount_path": mount_path}
|
|
893
|
+
if include_auto_commit and vol._auto_commit_on_ready:
|
|
894
|
+
ref["auto_commit"] = True
|
|
895
|
+
if include_auto_commit and vol._auto_commit_every:
|
|
896
|
+
ref["auto_commit_interval_seconds"] = vol._auto_commit_every
|
|
897
|
+
if vol._read_only:
|
|
898
|
+
ref["read_only"] = True
|
|
899
|
+
refs.append(ref)
|
|
900
|
+
return refs
|
|
901
|
+
|
|
902
|
+
def _image_manifest(self) -> dict:
|
|
903
|
+
img = self.image if self.image is not None else self._app.default_image
|
|
904
|
+
if img is None:
|
|
905
|
+
return {}
|
|
906
|
+
from ._image import Image
|
|
907
|
+
|
|
908
|
+
if isinstance(img, Image):
|
|
909
|
+
return {
|
|
910
|
+
"image_definition": img.to_definition(),
|
|
911
|
+
"definition_hash": img.definition_hash(),
|
|
912
|
+
}
|
|
913
|
+
from ._image import to_wire_image
|
|
914
|
+
|
|
915
|
+
return {"image": to_wire_image(img)}
|
|
916
|
+
|
|
917
|
+
def _manifest_function(self) -> dict:
|
|
918
|
+
data: dict[str, Any] = {
|
|
919
|
+
"name": self.__name__,
|
|
920
|
+
"function": self._function_ref(),
|
|
921
|
+
"resources": self._resource_spec(),
|
|
922
|
+
}
|
|
923
|
+
if self.class_name:
|
|
924
|
+
data["class_name"] = self.class_name
|
|
925
|
+
image = self._image_manifest()
|
|
926
|
+
if image:
|
|
927
|
+
data.update(image)
|
|
928
|
+
secrets = self._secret_name_refs()
|
|
929
|
+
if secrets:
|
|
930
|
+
data["secrets"] = secrets
|
|
931
|
+
volumes = self._volume_refs()
|
|
932
|
+
if volumes:
|
|
933
|
+
data["volumes"] = volumes
|
|
934
|
+
bucket_mounts = self._bucket_mount_specs()
|
|
935
|
+
if bucket_mounts:
|
|
936
|
+
data["bucket_mounts"] = bucket_mounts
|
|
937
|
+
if self._max_concurrent_inputs is not None:
|
|
938
|
+
data["max_concurrent_inputs"] = self._max_concurrent_inputs
|
|
939
|
+
if self.batch_config is not None:
|
|
940
|
+
data["batch_config"] = self.batch_config
|
|
941
|
+
return data
|
|
942
|
+
|
|
943
|
+
def _cron_spec(self) -> Optional[dict]:
|
|
944
|
+
"""Serialize this function's cron schedule for the deployment request.
|
|
945
|
+
|
|
946
|
+
Returns None if the function has no ``schedule=bladekit.Cron(...)``. The
|
|
947
|
+
server (handleCreateDeployment) inserts a cron_schedules row from this
|
|
948
|
+
(scheduler-v2.md §6, map-protocol.md is unrelated).
|
|
949
|
+
"""
|
|
950
|
+
if self.schedule is None:
|
|
951
|
+
return None
|
|
952
|
+
res = self._resource_spec()
|
|
953
|
+
spec: dict[str, Any] = {
|
|
954
|
+
"name": self.__name__,
|
|
955
|
+
"cron_expr": self.schedule.expression,
|
|
956
|
+
"overlap_policy": self.schedule.overlap_policy,
|
|
957
|
+
"function": self._function_ref(),
|
|
958
|
+
"resources": {
|
|
959
|
+
"gpu_type": res["gpu_type"],
|
|
960
|
+
"gpu_count": res["gpu_count"],
|
|
961
|
+
"cpu_millicores": res["cpu_millicores"],
|
|
962
|
+
"memory_mb": res["memory_mb"],
|
|
963
|
+
"timeout_seconds": res["timeout_seconds"],
|
|
964
|
+
},
|
|
965
|
+
}
|
|
966
|
+
if self.priority is not None:
|
|
967
|
+
spec["priority"] = self.priority
|
|
968
|
+
if self.namespace is not None:
|
|
969
|
+
spec["namespace"] = self.namespace
|
|
970
|
+
return spec
|
|
971
|
+
|
|
972
|
+
def _endpoint_specs(self) -> list[dict]:
|
|
973
|
+
specs = []
|
|
974
|
+
for decl in self._endpoints:
|
|
975
|
+
autoscaling = _autoscaling_spec(
|
|
976
|
+
min_replicas=decl.min_replicas,
|
|
977
|
+
max_replicas=decl.max_replicas,
|
|
978
|
+
max_concurrent_inputs=(
|
|
979
|
+
decl.max_concurrent_inputs or self._max_concurrent_inputs
|
|
980
|
+
),
|
|
981
|
+
scale_down_idle_seconds=decl.scale_down_idle_seconds,
|
|
982
|
+
max_queue_depth=decl.max_queue_depth,
|
|
983
|
+
reserved_replicas=decl.reserved_replicas,
|
|
984
|
+
)
|
|
985
|
+
# Autoscaling v2: fold the endpoint's Autoscale and the
|
|
986
|
+
# function-level @app.concurrent(target_inputs=) into the wire
|
|
987
|
+
# policy ({"v":1, ...}); None when neither is declared (v1).
|
|
988
|
+
policy = _scaling_policy_spec(
|
|
989
|
+
autoscale=decl.autoscale,
|
|
990
|
+
fn_target_inputs=self._target_inputs,
|
|
991
|
+
max_concurrent_inputs=autoscaling["max_concurrent_inputs"],
|
|
992
|
+
kind=decl.kind,
|
|
993
|
+
)
|
|
994
|
+
if policy is not None:
|
|
995
|
+
autoscaling["scaling_policy"] = policy
|
|
996
|
+
spec = {
|
|
997
|
+
"name": decl.name or self.__name__,
|
|
998
|
+
"kind": decl.kind,
|
|
999
|
+
"function": self._function_ref(),
|
|
1000
|
+
"path": decl.path,
|
|
1001
|
+
"methods": list(decl.methods),
|
|
1002
|
+
"resources": self._resource_spec(),
|
|
1003
|
+
"autoscaling": autoscaling,
|
|
1004
|
+
}
|
|
1005
|
+
# Deploy path: carry the function's volume mounts so replicas mount
|
|
1006
|
+
# them (create-or-get volumes that ask for it). Mirrors the run path.
|
|
1007
|
+
# auto_commit_on_ready rides only on local-server kinds (the commit
|
|
1008
|
+
# fires on serving-ready) — validated here, mirror of the server rule.
|
|
1009
|
+
volumes = self._volume_refs(include_auto_commit=True)
|
|
1010
|
+
if decl.kind not in ("web_server", "asgi_app"):
|
|
1011
|
+
for v in volumes:
|
|
1012
|
+
if v.get("auto_commit"):
|
|
1013
|
+
raise exc.InvalidArgumentError(
|
|
1014
|
+
"INVALID_ARGUMENT",
|
|
1015
|
+
f"volumes[{v['mount_path']!r}]: auto_commit_on_ready "
|
|
1016
|
+
f"requires a web_server or asgi_app endpoint, not "
|
|
1017
|
+
f"{decl.kind!r}",
|
|
1018
|
+
)
|
|
1019
|
+
if v.get("auto_commit_interval_seconds"):
|
|
1020
|
+
raise exc.InvalidArgumentError(
|
|
1021
|
+
"INVALID_ARGUMENT",
|
|
1022
|
+
f"volumes[{v['mount_path']!r}]: auto_commit_every "
|
|
1023
|
+
f"requires a web_server or asgi_app endpoint, not "
|
|
1024
|
+
f"{decl.kind!r}",
|
|
1025
|
+
)
|
|
1026
|
+
if volumes:
|
|
1027
|
+
spec["volumes"] = volumes
|
|
1028
|
+
# Carry the function's bucket mounts so each replica mounts them
|
|
1029
|
+
# (mirrors the run path). The backing secret rides in `secrets` below.
|
|
1030
|
+
bucket_mounts = self._bucket_mount_specs()
|
|
1031
|
+
if bucket_mounts:
|
|
1032
|
+
spec["bucket_mounts"] = bucket_mounts
|
|
1033
|
+
# Carry the function's secrets so each replica gets them injected as
|
|
1034
|
+
# env vars (mirrors the run path). Without this, secrets on a
|
|
1035
|
+
# web_server/asgi_app/web_endpoint deployment are silently dropped.
|
|
1036
|
+
secrets = self._secret_name_refs()
|
|
1037
|
+
if secrets:
|
|
1038
|
+
spec["secrets"] = secrets
|
|
1039
|
+
if decl.public:
|
|
1040
|
+
spec["public"] = True
|
|
1041
|
+
# §13.5: local HTTP server endpoints (web_server / asgi_app). The key
|
|
1042
|
+
# names MUST match openapi EndpointSpec (server_port /
|
|
1043
|
+
# startup_timeout_seconds). Emitted only when set, mirroring `public`.
|
|
1044
|
+
if decl.port is not None:
|
|
1045
|
+
spec["server_port"] = decl.port
|
|
1046
|
+
if decl.startup_timeout_seconds is not None:
|
|
1047
|
+
spec["startup_timeout_seconds"] = decl.startup_timeout_seconds
|
|
1048
|
+
# GPU snapshot opt-in (coldstart-snapshot.md §5). Emitted only when
|
|
1049
|
+
# True (mirrors `public`). Local mirror of the server rule: valid
|
|
1050
|
+
# ONLY on web_server with gpu_count==1 — fail here, before any
|
|
1051
|
+
# network call, instead of on a late 400 INVALID_ARGUMENT.
|
|
1052
|
+
if decl.snapshot:
|
|
1053
|
+
_, gpu_count = self._parse_gpu()
|
|
1054
|
+
if decl.kind != "web_server" or gpu_count != 1:
|
|
1055
|
+
raise exc.InvalidArgumentError(
|
|
1056
|
+
"INVALID_ARGUMENT",
|
|
1057
|
+
"snapshot=True requires web_server with a single GPU "
|
|
1058
|
+
"(gpu_count==1)",
|
|
1059
|
+
)
|
|
1060
|
+
spec["snapshot"] = True
|
|
1061
|
+
# deployed endpoint's capacity class (m8-spot.md §13.1/§13.6,
|
|
1062
|
+
# EndpointSpec.gpu_class). Omitted when default ("spot"), mirroring
|
|
1063
|
+
# how `public` is only emitted when non-default.
|
|
1064
|
+
if self.gpu_class != "spot":
|
|
1065
|
+
spec["gpu_class"] = self.gpu_class
|
|
1066
|
+
# dynamic batching config (batching.md §4). The runner reads it from
|
|
1067
|
+
# the deployed endpoint to fuse concurrent requests into batches.
|
|
1068
|
+
if self.batch_config is not None:
|
|
1069
|
+
spec["batch_config"] = self.batch_config
|
|
1070
|
+
specs.append(spec)
|
|
1071
|
+
return specs
|
|
1072
|
+
|
|
1073
|
+
def _enqueue_call(self, args: tuple, kwargs: dict) -> tuple[str, str]:
|
|
1074
|
+
from ._serialize import dumps_args, make_payload_in
|
|
1075
|
+
|
|
1076
|
+
client = self._app._require_client()
|
|
1077
|
+
|
|
1078
|
+
# Serialize and UPLOAD BEFORE the run exists. A run's idle_timeout_seconds
|
|
1079
|
+
# starts running against it the moment it can accept work, so anything done
|
|
1080
|
+
# between creating the run and posting the call is charged to that budget.
|
|
1081
|
+
# An out-of-line payload is a PUT of the caller's arguments to the object
|
|
1082
|
+
# store over the caller's uplink: 7 MB of audio measured 1.2s on a good
|
|
1083
|
+
# link and 28s on a normal one, and the ordering made that difference a
|
|
1084
|
+
# lifecycle risk for the run rather than just latency. The payload is
|
|
1085
|
+
# org-scoped and only bound to a run when the call is created, so it does
|
|
1086
|
+
# not need the run to exist yet.
|
|
1087
|
+
raw = dumps_args(args, kwargs)
|
|
1088
|
+
payload_in, object_bytes = make_payload_in(raw)
|
|
1089
|
+
if object_bytes is not None:
|
|
1090
|
+
decl = client.create_payload(payload_in["size_bytes"], payload_in["sha256"])
|
|
1091
|
+
client.upload_to_presigned(
|
|
1092
|
+
decl["put_url"], object_bytes, "application/octet-stream"
|
|
1093
|
+
)
|
|
1094
|
+
payload_in["payload_id"] = decl["payload_id"]
|
|
1095
|
+
|
|
1096
|
+
run_id = self._ensure_run()
|
|
1097
|
+
|
|
1098
|
+
try:
|
|
1099
|
+
call = client.create_call(run_id, payload_in)
|
|
1100
|
+
except exc.ConflictError as e:
|
|
1101
|
+
if e.code == "RUN_NOT_ACCEPTING_CALLS":
|
|
1102
|
+
# Race: the run failed at startup (IMAGE_PULL_FAILED,
|
|
1103
|
+
# CONTAINER_FAILED, PYTHON_VERSION_MISMATCH, …) between
|
|
1104
|
+
# _ensure_run and create_call. The stub and control-plane both
|
|
1105
|
+
# return RUN_NOT_ACCEPTING_CALLS when a call is posted to a
|
|
1106
|
+
# terminal run. Fetch the run's actual error and raise the
|
|
1107
|
+
# correct, actionable RunError instead of the generic conflict.
|
|
1108
|
+
self._check_run_terminal(client, run_id)
|
|
1109
|
+
# If _check_run_terminal did not raise (e.g. the run moved to
|
|
1110
|
+
# succeeded or back to a live state), propagate the original
|
|
1111
|
+
# error.
|
|
1112
|
+
raise
|
|
1113
|
+
return run_id, call["call_id"]
|
|
1114
|
+
|
|
1115
|
+
# -- public invocation --------------------------------------------------
|
|
1116
|
+
|
|
1117
|
+
def spawn(self, *args: Any, **kwargs: Any) -> FunctionCall:
|
|
1118
|
+
"""Fire-and-forget: enqueue a call, return a handle (non-blocking).
|
|
1119
|
+
|
|
1120
|
+
IMPORTANT — semantics differ from Modal. ``.spawn()`` appends a
|
|
1121
|
+
call to the SAME run (and therefore the SAME hot container) as every
|
|
1122
|
+
other ``.remote()`` / ``.spawn()`` of this function in this process.
|
|
1123
|
+
Calls run **FIFO, one at a time** on that one container — ``.spawn()``
|
|
1124
|
+
does NOT start a parallel container. So two ``.spawn()`` calls do not run
|
|
1125
|
+
concurrently; the second waits for the first. (True fan-out across the
|
|
1126
|
+
fleet — concurrent containers — arrives with ``Function.map()``.)
|
|
1127
|
+
"""
|
|
1128
|
+
run_id, call_id = self._enqueue_call(args, kwargs)
|
|
1129
|
+
return FunctionCall(self, run_id, call_id)
|
|
1130
|
+
|
|
1131
|
+
def remote(self, *args: Any, **kwargs: Any) -> Any:
|
|
1132
|
+
"""Blocking remote call: returns the result or re-raises the remote error."""
|
|
1133
|
+
run_id, call_id = self._enqueue_call(args, kwargs)
|
|
1134
|
+
return self._await_call(call_id, run_id, timeout=None)
|
|
1135
|
+
|
|
1136
|
+
def map(
|
|
1137
|
+
self,
|
|
1138
|
+
iterable: Any,
|
|
1139
|
+
*,
|
|
1140
|
+
order: str = "index",
|
|
1141
|
+
return_exceptions: bool = False,
|
|
1142
|
+
retries: Optional[dict] = None,
|
|
1143
|
+
input_timeout_seconds: Optional[int] = None,
|
|
1144
|
+
priority: Optional[int] = None,
|
|
1145
|
+
namespace: Optional[str] = None,
|
|
1146
|
+
max_concurrency: Optional[int] = None,
|
|
1147
|
+
):
|
|
1148
|
+
"""Fan-out ``f(x)`` over an iterable on the GPU fleet.
|
|
1149
|
+
|
|
1150
|
+
Yields results in the same order as the input iterable by default
|
|
1151
|
+
(``order="index"``). Use ``order="arrival"`` for maximum throughput
|
|
1152
|
+
when order does not matter.
|
|
1153
|
+
|
|
1154
|
+
The batch is persisted server-side: a client disconnect does NOT lose
|
|
1155
|
+
work. Inputs are pushed in streaming chunks so a 1 M-item iterable
|
|
1156
|
+
never sits in memory.
|
|
1157
|
+
|
|
1158
|
+
Args:
|
|
1159
|
+
iterable: Any Python iterable of arguments. Each element is passed
|
|
1160
|
+
as the *sole positional argument* to the function.
|
|
1161
|
+
order: ``"index"`` (default) or ``"arrival"``.
|
|
1162
|
+
return_exceptions: if ``True``, exceptions are yielded in-place
|
|
1163
|
+
instead of raising; the generator never stops early. If
|
|
1164
|
+
``False`` (default), the first failed input raises immediately.
|
|
1165
|
+
retries: override the retry policy for this batch, e.g.
|
|
1166
|
+
``{"max_retries": 3, "backoff": {"initial_seconds": 1,
|
|
1167
|
+
"max_seconds": 60, "factor": 2.0}}``.
|
|
1168
|
+
input_timeout_seconds: per-input timeout (distinct from the run
|
|
1169
|
+
global ``timeout=``).
|
|
1170
|
+
priority: scheduling priority 0–9 (9=urgent, 0=best-effort batch).
|
|
1171
|
+
Defaults to the function's priority or 5 (FIFO).
|
|
1172
|
+
namespace: override the namespace for quota accounting.
|
|
1173
|
+
max_concurrency: cap on concurrent worker runs for this batch.
|
|
1174
|
+
|
|
1175
|
+
Yields:
|
|
1176
|
+
The return value of the function for each input, in ``order`` order.
|
|
1177
|
+
"""
|
|
1178
|
+
# Wrap each item as (args=(item,), kwargs={}) for the wire format.
|
|
1179
|
+
def _items():
|
|
1180
|
+
for item in iterable:
|
|
1181
|
+
yield (item,), {}
|
|
1182
|
+
|
|
1183
|
+
yield from self._run_map(
|
|
1184
|
+
_items(),
|
|
1185
|
+
order=order,
|
|
1186
|
+
return_exceptions=return_exceptions,
|
|
1187
|
+
retries=retries,
|
|
1188
|
+
input_timeout_seconds=input_timeout_seconds,
|
|
1189
|
+
priority=priority,
|
|
1190
|
+
namespace=namespace,
|
|
1191
|
+
max_concurrency=max_concurrency,
|
|
1192
|
+
)
|
|
1193
|
+
|
|
1194
|
+
def starmap(
|
|
1195
|
+
self,
|
|
1196
|
+
iterable: Any,
|
|
1197
|
+
*,
|
|
1198
|
+
order: str = "index",
|
|
1199
|
+
return_exceptions: bool = False,
|
|
1200
|
+
retries: Optional[dict] = None,
|
|
1201
|
+
input_timeout_seconds: Optional[int] = None,
|
|
1202
|
+
priority: Optional[int] = None,
|
|
1203
|
+
namespace: Optional[str] = None,
|
|
1204
|
+
max_concurrency: Optional[int] = None,
|
|
1205
|
+
):
|
|
1206
|
+
"""Fan-out ``f(*args)`` over an iterable of arg tuples.
|
|
1207
|
+
|
|
1208
|
+
Like ``itertools.starmap``: each element of ``iterable`` is unpacked
|
|
1209
|
+
as positional arguments to the function.
|
|
1210
|
+
|
|
1211
|
+
Args:
|
|
1212
|
+
iterable: An iterable of tuples; each tuple is splatted as
|
|
1213
|
+
positional args. A tuple ``(a, b)`` maps to ``f(a, b)``.
|
|
1214
|
+
(remaining args: same as :meth:`map`)
|
|
1215
|
+
|
|
1216
|
+
Yields:
|
|
1217
|
+
The return value of the function for each input.
|
|
1218
|
+
"""
|
|
1219
|
+
def _items():
|
|
1220
|
+
for item in iterable:
|
|
1221
|
+
if not isinstance(item, tuple):
|
|
1222
|
+
item = tuple(item)
|
|
1223
|
+
yield item, {}
|
|
1224
|
+
|
|
1225
|
+
yield from self._run_map(
|
|
1226
|
+
_items(),
|
|
1227
|
+
order=order,
|
|
1228
|
+
return_exceptions=return_exceptions,
|
|
1229
|
+
retries=retries,
|
|
1230
|
+
input_timeout_seconds=input_timeout_seconds,
|
|
1231
|
+
priority=priority,
|
|
1232
|
+
namespace=namespace,
|
|
1233
|
+
max_concurrency=max_concurrency,
|
|
1234
|
+
)
|
|
1235
|
+
|
|
1236
|
+
def spawn_map(
|
|
1237
|
+
self,
|
|
1238
|
+
iterable: Any,
|
|
1239
|
+
*,
|
|
1240
|
+
retries: Optional[dict] = None,
|
|
1241
|
+
input_timeout_seconds: Optional[int] = None,
|
|
1242
|
+
priority: Optional[int] = None,
|
|
1243
|
+
namespace: Optional[str] = None,
|
|
1244
|
+
max_concurrency: Optional[int] = None,
|
|
1245
|
+
) -> "MapHandle":
|
|
1246
|
+
"""Fire an async fan-out of ``f(x)`` over an iterable; return a handle.
|
|
1247
|
+
|
|
1248
|
+
``spawn_map`` is to :meth:`map` what :meth:`spawn` is to :meth:`remote`:
|
|
1249
|
+
it combines ``spawn`` + ``map``. It creates the batch, pushes every
|
|
1250
|
+
input, and finalizes it server-side, then returns a
|
|
1251
|
+
:class:`~bladekit._map.MapHandle` IMMEDIATELY — it never blocks waiting for
|
|
1252
|
+
results. The batch is durable server-side, so you can poll it, walk away,
|
|
1253
|
+
or re-attach from another process with ``blade map resume <map_id>``.
|
|
1254
|
+
|
|
1255
|
+
Collect results later by iterating ``handle.results()`` (same ``order`` /
|
|
1256
|
+
``return_exceptions`` options as :meth:`map`), inspect progress with
|
|
1257
|
+
``handle.status()``, or stop it with ``handle.cancel()``.
|
|
1258
|
+
|
|
1259
|
+
Args:
|
|
1260
|
+
iterable: any iterable of arguments; each element is passed as the
|
|
1261
|
+
sole positional argument to the function (like :meth:`map`).
|
|
1262
|
+
(remaining args: same as :meth:`map`, minus ``order`` /
|
|
1263
|
+
``return_exceptions`` — those are chosen when reading results.)
|
|
1264
|
+
|
|
1265
|
+
Returns:
|
|
1266
|
+
A :class:`~bladekit._map.MapHandle` for the in-flight batch.
|
|
1267
|
+
"""
|
|
1268
|
+
def _items():
|
|
1269
|
+
for item in iterable:
|
|
1270
|
+
yield (item,), {}
|
|
1271
|
+
|
|
1272
|
+
return self._spawn_map(
|
|
1273
|
+
_items(),
|
|
1274
|
+
retries=retries,
|
|
1275
|
+
input_timeout_seconds=input_timeout_seconds,
|
|
1276
|
+
priority=priority,
|
|
1277
|
+
namespace=namespace,
|
|
1278
|
+
max_concurrency=max_concurrency,
|
|
1279
|
+
)
|
|
1280
|
+
|
|
1281
|
+
def spawn_starmap(
|
|
1282
|
+
self,
|
|
1283
|
+
iterable: Any,
|
|
1284
|
+
*,
|
|
1285
|
+
retries: Optional[dict] = None,
|
|
1286
|
+
input_timeout_seconds: Optional[int] = None,
|
|
1287
|
+
priority: Optional[int] = None,
|
|
1288
|
+
namespace: Optional[str] = None,
|
|
1289
|
+
max_concurrency: Optional[int] = None,
|
|
1290
|
+
) -> "MapHandle":
|
|
1291
|
+
"""Like :meth:`spawn_map`, but each element is splatted as positional args.
|
|
1292
|
+
|
|
1293
|
+
The ``starmap`` counterpart of :meth:`spawn_map`: a tuple ``(a, b)`` maps
|
|
1294
|
+
to ``f(a, b)``. Returns a :class:`~bladekit._map.MapHandle` without blocking.
|
|
1295
|
+
"""
|
|
1296
|
+
def _items():
|
|
1297
|
+
for item in iterable:
|
|
1298
|
+
if not isinstance(item, tuple):
|
|
1299
|
+
item = tuple(item)
|
|
1300
|
+
yield item, {}
|
|
1301
|
+
|
|
1302
|
+
return self._spawn_map(
|
|
1303
|
+
_items(),
|
|
1304
|
+
retries=retries,
|
|
1305
|
+
input_timeout_seconds=input_timeout_seconds,
|
|
1306
|
+
priority=priority,
|
|
1307
|
+
namespace=namespace,
|
|
1308
|
+
max_concurrency=max_concurrency,
|
|
1309
|
+
)
|
|
1310
|
+
|
|
1311
|
+
def _spawn_map(
|
|
1312
|
+
self,
|
|
1313
|
+
inputs,
|
|
1314
|
+
*,
|
|
1315
|
+
retries: Optional[dict],
|
|
1316
|
+
input_timeout_seconds: Optional[int],
|
|
1317
|
+
priority: Optional[int],
|
|
1318
|
+
namespace: Optional[str],
|
|
1319
|
+
max_concurrency: Optional[int],
|
|
1320
|
+
) -> "MapHandle":
|
|
1321
|
+
"""Shared implementation for spawn_map()/spawn_starmap()."""
|
|
1322
|
+
from ._map import spawn_map as _spawn_map_impl
|
|
1323
|
+
from ._serialize import cloudpickle_version
|
|
1324
|
+
|
|
1325
|
+
client = self._app._require_client()
|
|
1326
|
+
bundle_id = self._app._ensure_bundle()
|
|
1327
|
+
image_str, image_build_id = self._resolve_image()
|
|
1328
|
+
|
|
1329
|
+
eff_priority = priority if priority is not None else self.priority
|
|
1330
|
+
if eff_priority is not None:
|
|
1331
|
+
_validate_priority(eff_priority)
|
|
1332
|
+
eff_namespace = namespace if namespace is not None else self.namespace
|
|
1333
|
+
eff_retries = (
|
|
1334
|
+
_normalize_retries(retries) if retries is not None else self.retries
|
|
1335
|
+
)
|
|
1336
|
+
|
|
1337
|
+
return _spawn_map_impl(
|
|
1338
|
+
client,
|
|
1339
|
+
function_ref=self._function_ref(),
|
|
1340
|
+
bundle_id=bundle_id,
|
|
1341
|
+
resource_spec=self._resource_spec(),
|
|
1342
|
+
runtime={
|
|
1343
|
+
"python_version": _python_version(),
|
|
1344
|
+
"cloudpickle_version": cloudpickle_version(),
|
|
1345
|
+
},
|
|
1346
|
+
image=image_str,
|
|
1347
|
+
image_build_id=image_build_id,
|
|
1348
|
+
app_name=self._app.name if self._app.name else None,
|
|
1349
|
+
inputs=inputs,
|
|
1350
|
+
retries=eff_retries,
|
|
1351
|
+
input_timeout_seconds=input_timeout_seconds,
|
|
1352
|
+
priority=eff_priority,
|
|
1353
|
+
namespace=eff_namespace,
|
|
1354
|
+
max_concurrency=max_concurrency,
|
|
1355
|
+
gpu_class=self.gpu_class,
|
|
1356
|
+
best_effort=self.best_effort,
|
|
1357
|
+
max_interruptions=self.max_interruptions,
|
|
1358
|
+
)
|
|
1359
|
+
|
|
1360
|
+
def _run_map(
|
|
1361
|
+
self,
|
|
1362
|
+
inputs,
|
|
1363
|
+
*,
|
|
1364
|
+
order: str,
|
|
1365
|
+
return_exceptions: bool,
|
|
1366
|
+
retries: Optional[dict],
|
|
1367
|
+
input_timeout_seconds: Optional[int],
|
|
1368
|
+
priority: Optional[int],
|
|
1369
|
+
namespace: Optional[str],
|
|
1370
|
+
max_concurrency: Optional[int],
|
|
1371
|
+
):
|
|
1372
|
+
"""Shared implementation for map()/starmap()."""
|
|
1373
|
+
from ._map import run_map
|
|
1374
|
+
from ._serialize import cloudpickle_version
|
|
1375
|
+
|
|
1376
|
+
client = self._app._require_client()
|
|
1377
|
+
bundle_id = self._app._ensure_bundle()
|
|
1378
|
+
image_str, image_build_id = self._resolve_image()
|
|
1379
|
+
|
|
1380
|
+
# Priority: explicit argument > function default > contract default (5).
|
|
1381
|
+
eff_priority = priority if priority is not None else self.priority
|
|
1382
|
+
if eff_priority is not None:
|
|
1383
|
+
_validate_priority(eff_priority)
|
|
1384
|
+
|
|
1385
|
+
# Namespace: explicit argument > function default.
|
|
1386
|
+
eff_namespace = namespace if namespace is not None else self.namespace
|
|
1387
|
+
|
|
1388
|
+
# Retries: explicit argument > function default. Both accept an int
|
|
1389
|
+
# shorthand or a RetryPolicy dict (self.retries is already normalized).
|
|
1390
|
+
eff_retries = _normalize_retries(retries) if retries is not None else self.retries
|
|
1391
|
+
|
|
1392
|
+
yield from run_map(
|
|
1393
|
+
client,
|
|
1394
|
+
function_ref=self._function_ref(),
|
|
1395
|
+
bundle_id=bundle_id,
|
|
1396
|
+
resource_spec=self._resource_spec(),
|
|
1397
|
+
runtime={
|
|
1398
|
+
"python_version": _python_version(),
|
|
1399
|
+
"cloudpickle_version": cloudpickle_version(),
|
|
1400
|
+
},
|
|
1401
|
+
image=image_str,
|
|
1402
|
+
image_build_id=image_build_id,
|
|
1403
|
+
app_name=self._app.name if self._app.name else None,
|
|
1404
|
+
inputs=inputs,
|
|
1405
|
+
order=order,
|
|
1406
|
+
return_exceptions=return_exceptions,
|
|
1407
|
+
retries=eff_retries,
|
|
1408
|
+
input_timeout_seconds=input_timeout_seconds,
|
|
1409
|
+
priority=eff_priority,
|
|
1410
|
+
namespace=eff_namespace,
|
|
1411
|
+
max_concurrency=max_concurrency,
|
|
1412
|
+
# the batch's worker runs carry the function's capacity class
|
|
1413
|
+
# and preemption budget (m8-spot.md §13.6).
|
|
1414
|
+
gpu_class=self.gpu_class,
|
|
1415
|
+
best_effort=self.best_effort,
|
|
1416
|
+
max_interruptions=self.max_interruptions,
|
|
1417
|
+
)
|
|
1418
|
+
|
|
1419
|
+
# -- result resolution --------------------------------------------------
|
|
1420
|
+
|
|
1421
|
+
def _await_call(
|
|
1422
|
+
self, call_id: str, run_id: str, *, timeout: Optional[float]
|
|
1423
|
+
) -> Any:
|
|
1424
|
+
client = self._app._require_client()
|
|
1425
|
+
deadline = None if timeout is None else time.monotonic() + timeout
|
|
1426
|
+
interval = POLL_INITIAL_INTERVAL
|
|
1427
|
+
|
|
1428
|
+
while True:
|
|
1429
|
+
call = client.get_call(call_id)
|
|
1430
|
+
state = call.get("state")
|
|
1431
|
+
|
|
1432
|
+
if state in CALL_TERMINAL_STATES:
|
|
1433
|
+
return self._resolve_terminal_call(call, run_id)
|
|
1434
|
+
|
|
1435
|
+
# A cold-start infra failure (PYTHON_VERSION_MISMATCH, IMAGE_PULL_FAILED,
|
|
1436
|
+
# BUNDLE_FETCH_FAILED, CONTAINER_FAILED, NODE_LOST, …) fails the *run*
|
|
1437
|
+
# before any call is claimed, so the call stays `pending` forever
|
|
1438
|
+
# (schema.sql has no pending→failed call transition for infra). Poll the
|
|
1439
|
+
# run too and surface its error, otherwise `.remote(timeout=None)` hangs
|
|
1440
|
+
# instead of raising the clear error the contract promises (runner-
|
|
1441
|
+
# protocol.md §5). We only need this while the call hasn't been
|
|
1442
|
+
# claimed (`pending`): once `running`, the call itself becomes terminal
|
|
1443
|
+
# (running→failed via the agent handler) and carries the error.
|
|
1444
|
+
if state == "pending":
|
|
1445
|
+
self._check_run_terminal(client, run_id)
|
|
1446
|
+
|
|
1447
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
1448
|
+
raise exc.FabricError(
|
|
1449
|
+
f"timed out after {timeout}s waiting for call {call_id} "
|
|
1450
|
+
f"(state={state})"
|
|
1451
|
+
)
|
|
1452
|
+
|
|
1453
|
+
time.sleep(interval)
|
|
1454
|
+
interval = min(interval * 1.5, POLL_MAX_INTERVAL)
|
|
1455
|
+
|
|
1456
|
+
def _check_run_terminal(self, client: Any, run_id: str) -> None:
|
|
1457
|
+
"""Raise if the run reached a non-succeeded terminal state.
|
|
1458
|
+
|
|
1459
|
+
Used to break out of waiting on a call that will never be claimed because
|
|
1460
|
+
the run failed at startup. A run in ``succeeded`` with the call still
|
|
1461
|
+
non-terminal is left alone (the call result lands shortly after).
|
|
1462
|
+
"""
|
|
1463
|
+
run = client.get_run(run_id)
|
|
1464
|
+
run_state = run.get("state")
|
|
1465
|
+
if run_state == "lost":
|
|
1466
|
+
err = run.get("error") or {
|
|
1467
|
+
"code": "NODE_LOST",
|
|
1468
|
+
"message": "run lost before the call ran",
|
|
1469
|
+
}
|
|
1470
|
+
raise exc.run_lost_from_payload(err, run_id=run_id)
|
|
1471
|
+
if run_state == "failed":
|
|
1472
|
+
err = run.get("error")
|
|
1473
|
+
if err:
|
|
1474
|
+
raise exc.run_error_from_payload(err, run_id=run_id)
|
|
1475
|
+
raise exc.run_error_from_payload(
|
|
1476
|
+
{"code": "INTERNAL", "message": "run failed before the call ran"},
|
|
1477
|
+
run_id=run_id,
|
|
1478
|
+
)
|
|
1479
|
+
if run_state == "cancelled":
|
|
1480
|
+
raise exc.RunCancelled(run_id)
|
|
1481
|
+
|
|
1482
|
+
def _resolve_terminal_call(self, call: dict, run_id: str) -> Any:
|
|
1483
|
+
from ._serialize import decode_payload_out, loads
|
|
1484
|
+
|
|
1485
|
+
client = self._app._require_client()
|
|
1486
|
+
state = call.get("state")
|
|
1487
|
+
|
|
1488
|
+
if state == "succeeded":
|
|
1489
|
+
result = call.get("result")
|
|
1490
|
+
if result is None:
|
|
1491
|
+
return None
|
|
1492
|
+
raw = decode_payload_out(result, fetch=client.download_presigned)
|
|
1493
|
+
return loads(raw)
|
|
1494
|
+
|
|
1495
|
+
if state == "failed":
|
|
1496
|
+
# User exception vs infra error are mutually exclusive (openapi Call).
|
|
1497
|
+
user_exc = call.get("exception")
|
|
1498
|
+
if user_exc:
|
|
1499
|
+
raise self._build_remote_exception(user_exc, client)
|
|
1500
|
+
err = call.get("error")
|
|
1501
|
+
if err:
|
|
1502
|
+
raise exc.run_error_from_payload(err, run_id=run_id)
|
|
1503
|
+
raise exc.RunError("call failed without details", run_id=run_id)
|
|
1504
|
+
|
|
1505
|
+
if state == "cancelled":
|
|
1506
|
+
# A pending call is moved to `cancelled` when its RUN fails at
|
|
1507
|
+
# startup (IMAGE_PULL_FAILED, CONTAINER_FAILED, NODE_LOST, …) before
|
|
1508
|
+
# the call is ever claimed — the call carries no error then. Surface
|
|
1509
|
+
# the RUN's actual error so the user sees the real cause and a
|
|
1510
|
+
# non-zero exit, not a misleading "run cancelled" (GPU-PILOT-REPORT
|
|
1511
|
+
# UX debt). Only fall back to RunCancelled if the run was genuinely
|
|
1512
|
+
# cancelled (or has no failure recorded).
|
|
1513
|
+
self._check_run_terminal(client, run_id)
|
|
1514
|
+
raise exc.RunCancelled(run_id)
|
|
1515
|
+
|
|
1516
|
+
if state == "lost":
|
|
1517
|
+
err = call.get("error") or {"code": "NODE_LOST", "message": "call lost"}
|
|
1518
|
+
raise exc.run_lost_from_payload(err, run_id=run_id)
|
|
1519
|
+
|
|
1520
|
+
raise exc.FabricError(f"unexpected terminal call state {state!r}")
|
|
1521
|
+
|
|
1522
|
+
def _build_remote_exception(self, info: dict, client: Any) -> BaseException:
|
|
1523
|
+
"""Reconstruct (or wrap) the remote user exception.
|
|
1524
|
+
|
|
1525
|
+
Prefer re-raising the *original* exception (so user ``except`` clauses
|
|
1526
|
+
work) when ``pickled`` is present and unpickles; attach the remote
|
|
1527
|
+
traceback to its message. Otherwise raise the generic
|
|
1528
|
+
``RemoteFunctionError`` carrying the traceback.
|
|
1529
|
+
"""
|
|
1530
|
+
from ._serialize import decode_payload_out, loads
|
|
1531
|
+
|
|
1532
|
+
exc_type = info.get("type", "Exception")
|
|
1533
|
+
message = info.get("message", "")
|
|
1534
|
+
tb = info.get("traceback_str", "")
|
|
1535
|
+
|
|
1536
|
+
pickled = info.get("pickled")
|
|
1537
|
+
if pickled:
|
|
1538
|
+
try:
|
|
1539
|
+
raw = decode_payload_out(pickled, fetch=client.download_presigned)
|
|
1540
|
+
original = loads(raw)
|
|
1541
|
+
if isinstance(original, BaseException):
|
|
1542
|
+
# Surface the remote traceback on the re-raised exception so
|
|
1543
|
+
# it reads "as if local".
|
|
1544
|
+
original._forge_remote_traceback = tb # type: ignore[attr-defined]
|
|
1545
|
+
note = f"\n\n--- remote traceback ---\n{tb}"
|
|
1546
|
+
if hasattr(original, "add_note"):
|
|
1547
|
+
try:
|
|
1548
|
+
original.add_note(note) # py3.11+
|
|
1549
|
+
except Exception:
|
|
1550
|
+
pass
|
|
1551
|
+
return original
|
|
1552
|
+
except Exception:
|
|
1553
|
+
pass # fall through to generic wrapper
|
|
1554
|
+
|
|
1555
|
+
return exc.RemoteFunctionError(exc_type, message, tb)
|
|
1556
|
+
|
|
1557
|
+
|
|
1558
|
+
_MISSING = object()
|
|
1559
|
+
|
|
1560
|
+
|
|
1561
|
+
# function_calls.state → the async-handle status vocabulary (async-jobs-study.md
|
|
1562
|
+
# §3.3), aligned with Runpod/Replicate.
|
|
1563
|
+
_ASYNC_STATE_MAP = {
|
|
1564
|
+
"pending": "queued",
|
|
1565
|
+
"running": "running",
|
|
1566
|
+
"succeeded": "completed",
|
|
1567
|
+
"failed": "failed",
|
|
1568
|
+
"cancelled": "cancelled",
|
|
1569
|
+
"lost": "failed",
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1572
|
+
|
|
1573
|
+
class DeployedFunctionCall:
|
|
1574
|
+
"""Pollable handle to an async job spawned on a deployed endpoint.
|
|
1575
|
+
|
|
1576
|
+
Returned by ``Function.from_name(app, fn).spawn(...)`` (async-jobs-study.md).
|
|
1577
|
+
Re-attach from another process with ``DeployedFunctionCall.from_id(call_id)``.
|
|
1578
|
+
Distinct from ``FunctionCall`` (which decodes a cloudpickle result): here the
|
|
1579
|
+
result is the JSON body the deployed endpoint returned.
|
|
1580
|
+
"""
|
|
1581
|
+
|
|
1582
|
+
def __init__(
|
|
1583
|
+
self, client: Any, call_id: str, *, webhook_secret: Optional[str] = None
|
|
1584
|
+
) -> None:
|
|
1585
|
+
self._client = client
|
|
1586
|
+
self.call_id = call_id
|
|
1587
|
+
self.webhook_secret = webhook_secret # returned once if the CP generated it
|
|
1588
|
+
|
|
1589
|
+
def __repr__(self) -> str:
|
|
1590
|
+
return f"<bladekit.DeployedFunctionCall {self.call_id}>"
|
|
1591
|
+
|
|
1592
|
+
@classmethod
|
|
1593
|
+
def from_id(cls, call_id: str, *, client: Any = None) -> "DeployedFunctionCall":
|
|
1594
|
+
"""Re-attach to an async job by id from any process (parity Modal)."""
|
|
1595
|
+
if client is None:
|
|
1596
|
+
from ._client import Client
|
|
1597
|
+
from .config import load_config
|
|
1598
|
+
|
|
1599
|
+
cfg = load_config()
|
|
1600
|
+
if not cfg.token:
|
|
1601
|
+
raise exc.AuthenticationError(
|
|
1602
|
+
"UNAUTHENTICATED",
|
|
1603
|
+
"no token configured: run `blade token set <token>` or set FABRIC_TOKEN",
|
|
1604
|
+
)
|
|
1605
|
+
client = Client(cfg.api_url, cfg.token)
|
|
1606
|
+
return cls(client, call_id)
|
|
1607
|
+
|
|
1608
|
+
@property
|
|
1609
|
+
def object_id(self) -> str: # parity Modal
|
|
1610
|
+
return self.call_id
|
|
1611
|
+
|
|
1612
|
+
def status(self) -> str:
|
|
1613
|
+
"""Current status: queued|running|completed|failed|cancelled."""
|
|
1614
|
+
st = self._client.get_call(self.call_id).get("state", "")
|
|
1615
|
+
return _ASYNC_STATE_MAP.get(st, st)
|
|
1616
|
+
|
|
1617
|
+
def cancel(self) -> None:
|
|
1618
|
+
"""Cancel the job (best-effort; the shared replica keeps running)."""
|
|
1619
|
+
self._client.cancel_call(self.call_id)
|
|
1620
|
+
|
|
1621
|
+
def get(self, timeout: Optional[float] = None) -> Any:
|
|
1622
|
+
"""Block until terminal; return the endpoint's decoded JSON body.
|
|
1623
|
+
|
|
1624
|
+
Raises on a failed/cancelled/lost job. ``timeout`` bounds the wait in
|
|
1625
|
+
seconds (None waits indefinitely; 0 polls once then raises if not done).
|
|
1626
|
+
"""
|
|
1627
|
+
deadline = None if timeout is None else time.monotonic() + timeout
|
|
1628
|
+
interval = POLL_INITIAL_INTERVAL
|
|
1629
|
+
while True:
|
|
1630
|
+
call = self._client.get_call(self.call_id)
|
|
1631
|
+
state = call.get("state", "")
|
|
1632
|
+
if state in CALL_TERMINAL_STATES:
|
|
1633
|
+
if state != "succeeded":
|
|
1634
|
+
exc_info = call.get("exception") or {}
|
|
1635
|
+
err = call.get("error") or {}
|
|
1636
|
+
msg = (
|
|
1637
|
+
exc_info.get("message")
|
|
1638
|
+
or err.get("message")
|
|
1639
|
+
or f"async call {self.call_id} ended {state}"
|
|
1640
|
+
)
|
|
1641
|
+
code = err.get("code") or exc_info.get("type") or state.upper()
|
|
1642
|
+
raise exc.FabricError(f"{code}: {msg}")
|
|
1643
|
+
return _decode_async_body(call)
|
|
1644
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
1645
|
+
raise exc.TimeoutError_(
|
|
1646
|
+
f"async call {self.call_id} not finished within {timeout}s"
|
|
1647
|
+
)
|
|
1648
|
+
time.sleep(interval)
|
|
1649
|
+
interval = min(interval * 2, POLL_MAX_INTERVAL)
|
|
1650
|
+
|
|
1651
|
+
|
|
1652
|
+
def _decode_async_body(call: dict) -> Any:
|
|
1653
|
+
"""Decode http_response.body from a succeeded async call: inline_b64 XOR a
|
|
1654
|
+
presigned get_url (async-jobs-study.md §3.5). Returns parsed JSON, or the raw
|
|
1655
|
+
text if the body is not JSON, or None if empty."""
|
|
1656
|
+
import base64
|
|
1657
|
+
import json as _json
|
|
1658
|
+
|
|
1659
|
+
resp = call.get("http_response") or {}
|
|
1660
|
+
body = resp.get("body") or {}
|
|
1661
|
+
raw = b""
|
|
1662
|
+
if "inline_b64" in body:
|
|
1663
|
+
raw = base64.b64decode(body["inline_b64"])
|
|
1664
|
+
elif body.get("get_url"):
|
|
1665
|
+
import urllib.request
|
|
1666
|
+
|
|
1667
|
+
with urllib.request.urlopen(body["get_url"], timeout=60) as r: # noqa: S310
|
|
1668
|
+
raw = r.read()
|
|
1669
|
+
if not raw:
|
|
1670
|
+
return None
|
|
1671
|
+
try:
|
|
1672
|
+
return _json.loads(raw)
|
|
1673
|
+
except Exception:
|
|
1674
|
+
return raw.decode("utf-8", "replace")
|
|
1675
|
+
|
|
1676
|
+
|
|
1677
|
+
def _absolute_endpoint_url(endpoint: Optional[dict]) -> Optional[str]:
|
|
1678
|
+
"""The endpoint's own address, when the server reports an absolute one.
|
|
1679
|
+
|
|
1680
|
+
The server decides which ingress is live and says so in the endpoint's `url`:
|
|
1681
|
+
relative (`/serve/<app>/<path>`) while the path route is registered, absolute
|
|
1682
|
+
(`https://<app>.<domain><path>`) once it has been retired. Following it lets the
|
|
1683
|
+
SDK move with the deployment instead of hard-coding a route that may be gone, and
|
|
1684
|
+
an older control plane — which always reports the relative form — is unaffected.
|
|
1685
|
+
|
|
1686
|
+
None means the caller builds the path form, exactly as before. `request(path=...)`
|
|
1687
|
+
skips the endpoint resolve entirely, so it always takes that branch.
|
|
1688
|
+
"""
|
|
1689
|
+
url = (endpoint or {}).get("url")
|
|
1690
|
+
if isinstance(url, str) and url.startswith(("http://", "https://")):
|
|
1691
|
+
return url
|
|
1692
|
+
return None
|
|
1693
|
+
|
|
1694
|
+
|
|
1695
|
+
class _NamedFunction:
|
|
1696
|
+
def __init__(
|
|
1697
|
+
self, app_name: str, function_name: str, version: Optional[int] = None
|
|
1698
|
+
) -> None:
|
|
1699
|
+
if not isinstance(app_name, str) or not app_name:
|
|
1700
|
+
raise exc.InvalidArgumentError("INVALID_ARGUMENT", "app_name must be non-empty")
|
|
1701
|
+
if not isinstance(function_name, str) or not function_name:
|
|
1702
|
+
raise exc.InvalidArgumentError(
|
|
1703
|
+
"INVALID_ARGUMENT", "function_name must be non-empty"
|
|
1704
|
+
)
|
|
1705
|
+
if version is not None and (not isinstance(version, int) or isinstance(version, bool)):
|
|
1706
|
+
raise exc.InvalidArgumentError(
|
|
1707
|
+
"INVALID_ARGUMENT", "version must be an int (a deployment version)"
|
|
1708
|
+
)
|
|
1709
|
+
if version is not None and version < 1:
|
|
1710
|
+
raise exc.InvalidArgumentError(
|
|
1711
|
+
"INVALID_ARGUMENT", "version must be >= 1"
|
|
1712
|
+
)
|
|
1713
|
+
self.app_name = app_name
|
|
1714
|
+
self.function_name = function_name
|
|
1715
|
+
self.version = version
|
|
1716
|
+
self._client: Any = None
|
|
1717
|
+
self._endpoint: Optional[dict] = None
|
|
1718
|
+
|
|
1719
|
+
def __repr__(self) -> str:
|
|
1720
|
+
return f"<bladekit.Function.from_name {self.app_name}.{self.function_name}>"
|
|
1721
|
+
|
|
1722
|
+
def _require_client(self) -> Any:
|
|
1723
|
+
if self._client is None:
|
|
1724
|
+
from ._client import Client
|
|
1725
|
+
from .config import load_config
|
|
1726
|
+
|
|
1727
|
+
cfg = load_config()
|
|
1728
|
+
if not cfg.token:
|
|
1729
|
+
raise exc.AuthenticationError(
|
|
1730
|
+
"UNAUTHENTICATED",
|
|
1731
|
+
"no token configured: run `blade token set <token>` or set "
|
|
1732
|
+
"FABRIC_TOKEN",
|
|
1733
|
+
)
|
|
1734
|
+
self._client = Client(cfg.api_url, cfg.token)
|
|
1735
|
+
self._client.warn_if_outdated()
|
|
1736
|
+
return self._client
|
|
1737
|
+
|
|
1738
|
+
def _resolve_endpoint(self) -> dict:
|
|
1739
|
+
if self._endpoint is not None:
|
|
1740
|
+
return self._endpoint
|
|
1741
|
+
|
|
1742
|
+
client = self._require_client()
|
|
1743
|
+
app_obj = client.get_app_by_name(self.app_name)
|
|
1744
|
+
if app_obj.get("state") == "stopped":
|
|
1745
|
+
raise exc.UnavailableError(
|
|
1746
|
+
"APP_STOPPED", f"app {self.app_name!r} is stopped"
|
|
1747
|
+
)
|
|
1748
|
+
deployment = client.get_active_deployment_for_app(app_obj)
|
|
1749
|
+
# Resolve-and-VERIFY pinning. The resolved endpoint is cached for this
|
|
1750
|
+
# object's lifetime, so a long-lived caller that resolved before a
|
|
1751
|
+
# rollback keeps routing to whatever is active now — silently, and with a
|
|
1752
|
+
# different version of the code answering. version=N does not pin
|
|
1753
|
+
# EXECUTION to a dead deployment (nothing serves it); it turns that silent
|
|
1754
|
+
# drift into a deterministic error the caller can retry or fail on.
|
|
1755
|
+
if self.version is not None:
|
|
1756
|
+
active = deployment.get("version")
|
|
1757
|
+
if active != self.version:
|
|
1758
|
+
raise exc.NotFoundError(
|
|
1759
|
+
"DEPLOYMENT_VERSION_MISMATCH",
|
|
1760
|
+
f"app {self.app_name!r} is serving version {active!r}, not the "
|
|
1761
|
+
f"pinned version {self.version}; the app was redeployed or "
|
|
1762
|
+
"rolled back. Drop version= to follow the active deployment.",
|
|
1763
|
+
)
|
|
1764
|
+
endpoints = deployment.get("endpoints") or []
|
|
1765
|
+
if not endpoints:
|
|
1766
|
+
raise exc.NotFoundError(
|
|
1767
|
+
"DEPLOYMENT_NOT_FOUND",
|
|
1768
|
+
f"app {self.app_name!r} has no deployed web endpoints",
|
|
1769
|
+
)
|
|
1770
|
+
|
|
1771
|
+
selector = self.function_name
|
|
1772
|
+
matches = [
|
|
1773
|
+
ep for ep in endpoints if _endpoint_matches_selector(ep, selector)
|
|
1774
|
+
]
|
|
1775
|
+
if not matches:
|
|
1776
|
+
available = ", ".join(
|
|
1777
|
+
sorted(
|
|
1778
|
+
str(ep.get("name") or ep.get("path"))
|
|
1779
|
+
for ep in endpoints
|
|
1780
|
+
if ep.get("name") or ep.get("path")
|
|
1781
|
+
)
|
|
1782
|
+
)
|
|
1783
|
+
suffix = f"; available endpoints: {available}" if available else ""
|
|
1784
|
+
raise exc.NotFoundError(
|
|
1785
|
+
"NOT_FOUND",
|
|
1786
|
+
f"no web endpoint {selector!r} on deployed app {self.app_name!r}"
|
|
1787
|
+
f"{suffix}",
|
|
1788
|
+
)
|
|
1789
|
+
if len(matches) > 1:
|
|
1790
|
+
raise exc.InvalidArgumentError(
|
|
1791
|
+
"INVALID_ARGUMENT",
|
|
1792
|
+
f"endpoint selector {selector!r} is ambiguous; pass a path like "
|
|
1793
|
+
"'/predict' instead",
|
|
1794
|
+
)
|
|
1795
|
+
self._endpoint = matches[0]
|
|
1796
|
+
return self._endpoint
|
|
1797
|
+
|
|
1798
|
+
def request(
|
|
1799
|
+
self,
|
|
1800
|
+
*,
|
|
1801
|
+
method: str = "POST",
|
|
1802
|
+
path: Optional[str] = None,
|
|
1803
|
+
json: Any = None,
|
|
1804
|
+
headers: Optional[Mapping[str, str]] = None,
|
|
1805
|
+
params: Optional[Mapping[str, Any]] = None,
|
|
1806
|
+
) -> Any:
|
|
1807
|
+
"""Invoke the deployed web endpoint over HTTP and return JSON."""
|
|
1808
|
+
endpoint = None if path else self._resolve_endpoint()
|
|
1809
|
+
endpoint_path = path or str((endpoint or {}).get("path"))
|
|
1810
|
+
if not endpoint_path.startswith("/"):
|
|
1811
|
+
endpoint_path = f"/{endpoint_path}"
|
|
1812
|
+
try:
|
|
1813
|
+
return self._require_client().invoke_web_endpoint(
|
|
1814
|
+
self.app_name,
|
|
1815
|
+
endpoint_path,
|
|
1816
|
+
method=method,
|
|
1817
|
+
json=json,
|
|
1818
|
+
headers=headers,
|
|
1819
|
+
params=params,
|
|
1820
|
+
url=_absolute_endpoint_url(endpoint),
|
|
1821
|
+
)
|
|
1822
|
+
except exc.NotFoundError:
|
|
1823
|
+
# The cached endpoint went stale mid-flight (redeploy or rollback
|
|
1824
|
+
# dropped this path). Re-resolve ONCE against the now-active
|
|
1825
|
+
# deployment and retry. Only for a resolved endpoint: an explicit
|
|
1826
|
+
# path= is the caller's own routing, so its 404 is the real answer.
|
|
1827
|
+
#
|
|
1828
|
+
# Under version=, the re-resolve is what converts a bare 404 into the
|
|
1829
|
+
# DEPLOYMENT_VERSION_MISMATCH above, which says what actually changed.
|
|
1830
|
+
if path is not None or self._endpoint is None:
|
|
1831
|
+
raise
|
|
1832
|
+
self._endpoint = None
|
|
1833
|
+
retry_endpoint = self._resolve_endpoint()
|
|
1834
|
+
retry_path = str(retry_endpoint.get("path"))
|
|
1835
|
+
if not retry_path.startswith("/"):
|
|
1836
|
+
retry_path = f"/{retry_path}"
|
|
1837
|
+
# The re-resolved endpoint's OWN url, not the stale one's: the very
|
|
1838
|
+
# redeploy that invalidated the path is what retires a path route and
|
|
1839
|
+
# turns the server's reported url absolute, so reusing the first
|
|
1840
|
+
# endpoint's answer here would retry against the address that just
|
|
1841
|
+
# went away.
|
|
1842
|
+
return self._require_client().invoke_web_endpoint(
|
|
1843
|
+
self.app_name,
|
|
1844
|
+
retry_path,
|
|
1845
|
+
method=method,
|
|
1846
|
+
json=json,
|
|
1847
|
+
headers=headers,
|
|
1848
|
+
params=params,
|
|
1849
|
+
url=_absolute_endpoint_url(retry_endpoint),
|
|
1850
|
+
)
|
|
1851
|
+
|
|
1852
|
+
def web(
|
|
1853
|
+
self,
|
|
1854
|
+
payload: Any = _MISSING,
|
|
1855
|
+
*,
|
|
1856
|
+
method: str = "POST",
|
|
1857
|
+
path: Optional[str] = None,
|
|
1858
|
+
headers: Optional[Mapping[str, str]] = None,
|
|
1859
|
+
params: Optional[Mapping[str, Any]] = None,
|
|
1860
|
+
**json_fields: Any,
|
|
1861
|
+
) -> Any:
|
|
1862
|
+
return self.remote_json(
|
|
1863
|
+
payload,
|
|
1864
|
+
method=method,
|
|
1865
|
+
path=path,
|
|
1866
|
+
headers=headers,
|
|
1867
|
+
params=params,
|
|
1868
|
+
**json_fields,
|
|
1869
|
+
)
|
|
1870
|
+
|
|
1871
|
+
def remote_json(
|
|
1872
|
+
self,
|
|
1873
|
+
payload: Any = _MISSING,
|
|
1874
|
+
*,
|
|
1875
|
+
method: str = "POST",
|
|
1876
|
+
path: Optional[str] = None,
|
|
1877
|
+
headers: Optional[Mapping[str, str]] = None,
|
|
1878
|
+
params: Optional[Mapping[str, Any]] = None,
|
|
1879
|
+
**json_fields: Any,
|
|
1880
|
+
) -> Any:
|
|
1881
|
+
"""POST a JSON payload to the matching deployed web endpoint."""
|
|
1882
|
+
if payload is not _MISSING and json_fields:
|
|
1883
|
+
raise exc.InvalidArgumentError(
|
|
1884
|
+
"INVALID_ARGUMENT",
|
|
1885
|
+
"remote_json accepts either one payload or keyword JSON fields, not both",
|
|
1886
|
+
)
|
|
1887
|
+
body = json_fields if payload is _MISSING and json_fields else payload
|
|
1888
|
+
if body is _MISSING:
|
|
1889
|
+
body = None
|
|
1890
|
+
return self.request(
|
|
1891
|
+
method=method,
|
|
1892
|
+
path=path,
|
|
1893
|
+
json=body,
|
|
1894
|
+
headers=headers,
|
|
1895
|
+
params=params,
|
|
1896
|
+
)
|
|
1897
|
+
|
|
1898
|
+
def remote(self, *args: Any, **kwargs: Any) -> Any:
|
|
1899
|
+
"""Invoke a deployed web endpoint and return the JSON response.
|
|
1900
|
+
|
|
1901
|
+
Contractual path: the control plane routes the call through the web
|
|
1902
|
+
endpoint HTTP ingress (``/serve/<app>/<path>``, conventions.md §13).
|
|
1903
|
+
There is no cloudpickle-over-RPC endpoint for deployed functions in the
|
|
1904
|
+
current contract (see sdk/CONTRACT-ISSUES.md).
|
|
1905
|
+
|
|
1906
|
+
Calling convention:
|
|
1907
|
+
- A single positional arg is sent as the JSON body.
|
|
1908
|
+
- Keyword args are sent as the JSON body dict.
|
|
1909
|
+
- No args → empty JSON body (null).
|
|
1910
|
+
- Mixing positional and keyword args raises InvalidArgumentError.
|
|
1911
|
+
"""
|
|
1912
|
+
if args and kwargs:
|
|
1913
|
+
raise exc.InvalidArgumentError(
|
|
1914
|
+
"INVALID_ARGUMENT",
|
|
1915
|
+
"Function.from_name.remote() accepts either one positional arg "
|
|
1916
|
+
"or keyword args, not both. For full RPC semantics, see "
|
|
1917
|
+
"sdk/CONTRACT-ISSUES.md.",
|
|
1918
|
+
)
|
|
1919
|
+
if len(args) > 1:
|
|
1920
|
+
raise exc.InvalidArgumentError(
|
|
1921
|
+
"INVALID_ARGUMENT",
|
|
1922
|
+
"Function.from_name.remote() accepts at most one positional arg "
|
|
1923
|
+
"(the JSON body). For multi-arg calls, use keyword args or "
|
|
1924
|
+
"remote_json().",
|
|
1925
|
+
)
|
|
1926
|
+
body = args[0] if args else (kwargs if kwargs else None)
|
|
1927
|
+
return self.request(json=body)
|
|
1928
|
+
|
|
1929
|
+
def spawn(
|
|
1930
|
+
self, *args: Any, webhook: "Any" = None, **kwargs: Any
|
|
1931
|
+
) -> "DeployedFunctionCall":
|
|
1932
|
+
"""Invoke a deployed endpoint ASYNCHRONOUSLY; return a pollable handle.
|
|
1933
|
+
|
|
1934
|
+
Route: ``POST /v1/serve-async/<app>/<path>`` — the control plane enqueues
|
|
1935
|
+
an HTTP function call and answers ``202`` with a ``call_id`` WITHOUT
|
|
1936
|
+
holding the connection (async-jobs-study.md). Poll the returned handle
|
|
1937
|
+
with ``.get()`` / ``.status()``, or re-attach from another process with
|
|
1938
|
+
``DeployedFunctionCall.from_id(call_id)``.
|
|
1939
|
+
|
|
1940
|
+
JSON-only, same body convention as ``remote()``: one positional arg OR
|
|
1941
|
+
keyword args become the JSON body sent to the endpoint. ``webhook=`` (a
|
|
1942
|
+
``bladekit.Webhook``, dict, or bare URL str) registers a completion webhook;
|
|
1943
|
+
if it is signed and the CP generated the secret, it comes back once on
|
|
1944
|
+
``handle.webhook_secret``. Setup errors (app not found, no active
|
|
1945
|
+
deployment, ambiguous selector) propagate.
|
|
1946
|
+
|
|
1947
|
+
NOTE: this REPLACES the previous fire-and-forget ``spawn`` (which returned
|
|
1948
|
+
``None`` and discarded the response) — hence the 0.5.0 contract bump.
|
|
1949
|
+
"""
|
|
1950
|
+
if args and kwargs:
|
|
1951
|
+
raise exc.InvalidArgumentError(
|
|
1952
|
+
"INVALID_ARGUMENT",
|
|
1953
|
+
"Function.from_name.spawn() accepts either one positional arg "
|
|
1954
|
+
"or keyword args, not both.",
|
|
1955
|
+
)
|
|
1956
|
+
if len(args) > 1:
|
|
1957
|
+
raise exc.InvalidArgumentError(
|
|
1958
|
+
"INVALID_ARGUMENT",
|
|
1959
|
+
"Function.from_name.spawn() accepts at most one positional arg.",
|
|
1960
|
+
)
|
|
1961
|
+
from ._webhook import _webhook_dict
|
|
1962
|
+
|
|
1963
|
+
body = args[0] if args else (kwargs if kwargs else None)
|
|
1964
|
+
# Resolve the endpoint eagerly: NotFoundError, UnavailableError
|
|
1965
|
+
# (APP_STOPPED) and InvalidArgumentError (ambiguous selector) must
|
|
1966
|
+
# propagate so the caller learns they misfired.
|
|
1967
|
+
endpoint = self._resolve_endpoint()
|
|
1968
|
+
endpoint_path = str(endpoint.get("path"))
|
|
1969
|
+
if not endpoint_path.startswith("/"):
|
|
1970
|
+
endpoint_path = f"/{endpoint_path}"
|
|
1971
|
+
client = self._require_client()
|
|
1972
|
+
resp = client.spawn_async_endpoint(
|
|
1973
|
+
self.app_name, endpoint_path, input=body, webhook=_webhook_dict(webhook)
|
|
1974
|
+
)
|
|
1975
|
+
return DeployedFunctionCall(
|
|
1976
|
+
client, resp["call_id"], webhook_secret=(resp.get("webhook") or {}).get("secret")
|
|
1977
|
+
)
|
|
1978
|
+
|
|
1979
|
+
|
|
1980
|
+
def _endpoint_matches_selector(endpoint: dict, selector: str) -> bool:
|
|
1981
|
+
path = endpoint.get("path")
|
|
1982
|
+
if selector.startswith("/"):
|
|
1983
|
+
return path == selector
|
|
1984
|
+
|
|
1985
|
+
if endpoint.get("name") == selector:
|
|
1986
|
+
return True
|
|
1987
|
+
|
|
1988
|
+
function_ref = endpoint.get("function") or {}
|
|
1989
|
+
qualname = str(function_ref.get("qualname") or "")
|
|
1990
|
+
if qualname == selector or qualname.rsplit(".", 1)[-1] == selector:
|
|
1991
|
+
return True
|
|
1992
|
+
|
|
1993
|
+
if isinstance(path, str) and path.lstrip("/") == selector:
|
|
1994
|
+
return True
|
|
1995
|
+
return False
|
|
1996
|
+
|
|
1997
|
+
|
|
1998
|
+
_GPU_CLASSES = ("spot", "reserved")
|
|
1999
|
+
|
|
2000
|
+
|
|
2001
|
+
def _validate_gpu_class(gpu_class: str) -> None:
|
|
2002
|
+
"""Validate gpu_class ∈ {spot, reserved} (m8-spot.md §13.1)."""
|
|
2003
|
+
if gpu_class not in _GPU_CLASSES:
|
|
2004
|
+
raise exc.InvalidArgumentError(
|
|
2005
|
+
"INVALID_ARGUMENT",
|
|
2006
|
+
f"gpu_class must be one of {_GPU_CLASSES} (spot = cheap/preemptible, "
|
|
2007
|
+
f"reserved = no preemption); got {gpu_class!r}",
|
|
2008
|
+
)
|
|
2009
|
+
|
|
2010
|
+
|
|
2011
|
+
def _validate_priority(priority: int) -> None:
|
|
2012
|
+
"""Validate priority is in [PRIORITY_MIN, PRIORITY_MAX] (conventions.md §6)."""
|
|
2013
|
+
if not isinstance(priority, int) or priority < PRIORITY_MIN or priority > PRIORITY_MAX:
|
|
2014
|
+
raise exc.InvalidArgumentError(
|
|
2015
|
+
"PRIORITY_INVALID",
|
|
2016
|
+
f"priority must be an integer between {PRIORITY_MIN} and {PRIORITY_MAX} "
|
|
2017
|
+
f"(higher = more urgent); got {priority!r}",
|
|
2018
|
+
)
|
|
2019
|
+
|
|
2020
|
+
|
|
2021
|
+
def _normalize_retries(retries):
|
|
2022
|
+
"""Accept ``retries`` as an int shorthand (Modal-style ``retries=3``) or a
|
|
2023
|
+
full RetryPolicy dict; return the validated dict form (or None)."""
|
|
2024
|
+
if retries is None:
|
|
2025
|
+
return None
|
|
2026
|
+
if isinstance(retries, bool):
|
|
2027
|
+
raise exc.InvalidArgumentError(
|
|
2028
|
+
"RETRY_POLICY_INVALID", "retries must be an int or a dict, not a bool"
|
|
2029
|
+
)
|
|
2030
|
+
if isinstance(retries, int):
|
|
2031
|
+
retries = {"max_retries": retries}
|
|
2032
|
+
_validate_retries(retries)
|
|
2033
|
+
return retries
|
|
2034
|
+
|
|
2035
|
+
|
|
2036
|
+
def _validate_retries(retries: dict) -> None:
|
|
2037
|
+
"""Validate a RetryPolicy dict (conventions.md §6)."""
|
|
2038
|
+
if not isinstance(retries, dict):
|
|
2039
|
+
raise exc.InvalidArgumentError(
|
|
2040
|
+
"RETRY_POLICY_INVALID",
|
|
2041
|
+
f"retries must be an int (e.g. retries=3) or a RetryPolicy dict, "
|
|
2042
|
+
f"got {type(retries).__name__}",
|
|
2043
|
+
)
|
|
2044
|
+
max_retries = retries.get("max_retries", 0)
|
|
2045
|
+
if not isinstance(max_retries, int) or max_retries < 0 or max_retries > RETRIES_MAX:
|
|
2046
|
+
raise exc.InvalidArgumentError(
|
|
2047
|
+
"RETRY_POLICY_INVALID",
|
|
2048
|
+
f"retries.max_retries must be an integer between 0 and {RETRIES_MAX}; "
|
|
2049
|
+
f"got {max_retries!r}",
|
|
2050
|
+
)
|
|
2051
|
+
backoff = retries.get("backoff")
|
|
2052
|
+
if backoff is not None:
|
|
2053
|
+
if not isinstance(backoff, dict):
|
|
2054
|
+
raise exc.InvalidArgumentError(
|
|
2055
|
+
"RETRY_POLICY_INVALID", "retries.backoff must be a dict"
|
|
2056
|
+
)
|
|
2057
|
+
initial = backoff.get("initial_seconds", 1)
|
|
2058
|
+
if not isinstance(initial, (int, float)) or initial < 0 or initial > RETRIES_BACKOFF_INITIAL_MAX:
|
|
2059
|
+
raise exc.InvalidArgumentError(
|
|
2060
|
+
"RETRY_POLICY_INVALID",
|
|
2061
|
+
f"retries.backoff.initial_seconds must be between 0 and "
|
|
2062
|
+
f"{RETRIES_BACKOFF_INITIAL_MAX}; got {initial!r}",
|
|
2063
|
+
)
|
|
2064
|
+
max_s = backoff.get("max_seconds", 60)
|
|
2065
|
+
if not isinstance(max_s, (int, float)) or max_s < 0 or max_s > RETRIES_BACKOFF_MAX_MAX:
|
|
2066
|
+
raise exc.InvalidArgumentError(
|
|
2067
|
+
"RETRY_POLICY_INVALID",
|
|
2068
|
+
f"retries.backoff.max_seconds must be between 0 and "
|
|
2069
|
+
f"{RETRIES_BACKOFF_MAX_MAX}; got {max_s!r}",
|
|
2070
|
+
)
|
|
2071
|
+
factor = backoff.get("factor", 2.0)
|
|
2072
|
+
if not isinstance(factor, (int, float)) or factor < RETRIES_BACKOFF_FACTOR_MIN or factor > RETRIES_BACKOFF_FACTOR_MAX:
|
|
2073
|
+
raise exc.InvalidArgumentError(
|
|
2074
|
+
"RETRY_POLICY_INVALID",
|
|
2075
|
+
f"retries.backoff.factor must be between {RETRIES_BACKOFF_FACTOR_MIN} "
|
|
2076
|
+
f"and {RETRIES_BACKOFF_FACTOR_MAX}; got {factor!r}",
|
|
2077
|
+
)
|
|
2078
|
+
|
|
2079
|
+
|
|
2080
|
+
def _validate_app_name(name: str) -> None:
|
|
2081
|
+
if not isinstance(name, str) or not _APP_NAME_RE.match(name):
|
|
2082
|
+
raise exc.InvalidArgumentError(
|
|
2083
|
+
"APP_NAME_INVALID",
|
|
2084
|
+
f"invalid app name {name!r}; must match '^[a-z][a-z0-9-]{{2,62}}$'",
|
|
2085
|
+
)
|
|
2086
|
+
|
|
2087
|
+
|
|
2088
|
+
def _normalize_methods(
|
|
2089
|
+
methods: Optional[Sequence[str]], *, default: Sequence[str]
|
|
2090
|
+
) -> tuple[str, ...]:
|
|
2091
|
+
raw = tuple(default if methods is None else methods)
|
|
2092
|
+
if not raw:
|
|
2093
|
+
raise exc.InvalidArgumentError(
|
|
2094
|
+
"INVALID_ARGUMENT", "endpoint methods cannot be empty"
|
|
2095
|
+
)
|
|
2096
|
+
normalized = tuple(str(m).upper() for m in raw)
|
|
2097
|
+
bad = [m for m in normalized if m not in _HTTP_METHODS]
|
|
2098
|
+
if bad:
|
|
2099
|
+
raise exc.InvalidArgumentError(
|
|
2100
|
+
"INVALID_ARGUMENT",
|
|
2101
|
+
f"unsupported HTTP method(s): {', '.join(bad)}",
|
|
2102
|
+
)
|
|
2103
|
+
if len(normalized) > 8:
|
|
2104
|
+
raise exc.InvalidArgumentError(
|
|
2105
|
+
"INVALID_ARGUMENT", "endpoint methods are limited to 8 entries"
|
|
2106
|
+
)
|
|
2107
|
+
if len(set(normalized)) != len(normalized):
|
|
2108
|
+
raise exc.InvalidArgumentError(
|
|
2109
|
+
"INVALID_ARGUMENT", "endpoint methods must be unique"
|
|
2110
|
+
)
|
|
2111
|
+
return normalized
|
|
2112
|
+
|
|
2113
|
+
|
|
2114
|
+
def _validate_path(path: str) -> str:
|
|
2115
|
+
if not isinstance(path, str) or not path.startswith("/"):
|
|
2116
|
+
raise exc.InvalidArgumentError(
|
|
2117
|
+
"INVALID_ARGUMENT", f"endpoint path {path!r} must start with '/'"
|
|
2118
|
+
)
|
|
2119
|
+
if len(path) > 1024:
|
|
2120
|
+
raise exc.InvalidArgumentError(
|
|
2121
|
+
"INVALID_ARGUMENT", "endpoint path cannot exceed 1024 characters"
|
|
2122
|
+
)
|
|
2123
|
+
return path
|
|
2124
|
+
|
|
2125
|
+
|
|
2126
|
+
def _reject_pinned_server_port(port: Optional[int]) -> None:
|
|
2127
|
+
# §13.5: web_server is DYNAMIC-ONLY. The agent assigns each container a
|
|
2128
|
+
# node-unique loopback port via FORGE_SERVER_PORT (so co-located replicas can
|
|
2129
|
+
# never collide on a shared host loopback) and the user's server binds
|
|
2130
|
+
# $FABRIC_SERVER_PORT. Pinning a fixed port is no longer supported: it
|
|
2131
|
+
# re-introduced the co-location cross-talk bug this design exists to prevent.
|
|
2132
|
+
if port is not None:
|
|
2133
|
+
raise exc.InvalidArgumentError(
|
|
2134
|
+
"INVALID_ARGUMENT",
|
|
2135
|
+
"web_server no longer accepts a pinned port= (it caused co-located "
|
|
2136
|
+
"replicas to collide on the shared host loopback). Omit it and bind "
|
|
2137
|
+
"$FABRIC_SERVER_PORT — the agent assigns a unique port per container.",
|
|
2138
|
+
)
|
|
2139
|
+
|
|
2140
|
+
|
|
2141
|
+
def _validate_startup_timeout(value: int) -> None:
|
|
2142
|
+
# §13.5 / openapi EndpointSpec.startup_timeout_seconds: integer >= 10.
|
|
2143
|
+
#
|
|
2144
|
+
# The UPPER bound is the server's to enforce, not this SDK's. It used to be
|
|
2145
|
+
# a hardcoded 1800 here as well as there, and thirty minutes is under what a
|
|
2146
|
+
# first torch.compile over four ranks costs — so the client refused a
|
|
2147
|
+
# perfectly legal value before the request was ever made, which is the worst
|
|
2148
|
+
# place to be wrong: the deployment's real ceiling
|
|
2149
|
+
# (FORGE_MAX_STARTUP_TIMEOUT) is not something this process can know, and a
|
|
2150
|
+
# mirrored constant is guaranteed to disagree with some deployment. The
|
|
2151
|
+
# server answers 400 INVALID_ARGUMENT naming its own ceiling.
|
|
2152
|
+
if not isinstance(value, int) or isinstance(value, bool) or value < 10:
|
|
2153
|
+
raise exc.InvalidArgumentError(
|
|
2154
|
+
"INVALID_ARGUMENT",
|
|
2155
|
+
"startup_timeout must be an integer of at least 10 seconds",
|
|
2156
|
+
)
|
|
2157
|
+
|
|
2158
|
+
|
|
2159
|
+
def _validate_concurrency(value: int) -> None:
|
|
2160
|
+
if not isinstance(value, int) or value < 1 or value > 100:
|
|
2161
|
+
raise exc.InvalidArgumentError(
|
|
2162
|
+
"INVALID_ARGUMENT",
|
|
2163
|
+
"max_concurrent_inputs must be an integer between 1 and 100",
|
|
2164
|
+
)
|
|
2165
|
+
|
|
2166
|
+
|
|
2167
|
+
def _validate_autoscaling(
|
|
2168
|
+
*,
|
|
2169
|
+
min_replicas: Optional[int],
|
|
2170
|
+
max_replicas: Optional[int],
|
|
2171
|
+
max_concurrent_inputs: Optional[int],
|
|
2172
|
+
scale_down_idle_seconds: Optional[int],
|
|
2173
|
+
max_queue_depth: Optional[int] = None,
|
|
2174
|
+
) -> None:
|
|
2175
|
+
if min_replicas is not None and (
|
|
2176
|
+
not isinstance(min_replicas, int)
|
|
2177
|
+
or min_replicas < 0
|
|
2178
|
+
or min_replicas > 50
|
|
2179
|
+
):
|
|
2180
|
+
raise exc.InvalidArgumentError(
|
|
2181
|
+
"AUTOSCALE_LIMIT_INVALID", "min_replicas must be between 0 and 50"
|
|
2182
|
+
)
|
|
2183
|
+
if max_replicas is not None and (
|
|
2184
|
+
not isinstance(max_replicas, int)
|
|
2185
|
+
or max_replicas < 1
|
|
2186
|
+
or max_replicas > 50
|
|
2187
|
+
):
|
|
2188
|
+
raise exc.InvalidArgumentError(
|
|
2189
|
+
"AUTOSCALE_LIMIT_INVALID", "max_replicas must be between 1 and 50"
|
|
2190
|
+
)
|
|
2191
|
+
if (
|
|
2192
|
+
min_replicas is not None
|
|
2193
|
+
and max_replicas is not None
|
|
2194
|
+
and min_replicas > max_replicas
|
|
2195
|
+
):
|
|
2196
|
+
raise exc.InvalidArgumentError(
|
|
2197
|
+
"AUTOSCALE_LIMIT_INVALID", "min_replicas cannot exceed max_replicas"
|
|
2198
|
+
)
|
|
2199
|
+
if max_concurrent_inputs is not None:
|
|
2200
|
+
_validate_concurrency(max_concurrent_inputs)
|
|
2201
|
+
if scale_down_idle_seconds is not None and (
|
|
2202
|
+
not isinstance(scale_down_idle_seconds, int)
|
|
2203
|
+
or scale_down_idle_seconds < 30
|
|
2204
|
+
or scale_down_idle_seconds > 3600
|
|
2205
|
+
):
|
|
2206
|
+
raise exc.InvalidArgumentError(
|
|
2207
|
+
"AUTOSCALE_LIMIT_INVALID",
|
|
2208
|
+
"scale_down_idle_seconds must be between 30 and 3600",
|
|
2209
|
+
)
|
|
2210
|
+
if max_queue_depth is not None and (
|
|
2211
|
+
not isinstance(max_queue_depth, int)
|
|
2212
|
+
or isinstance(max_queue_depth, bool)
|
|
2213
|
+
or max_queue_depth < 0
|
|
2214
|
+
or max_queue_depth > 10000
|
|
2215
|
+
):
|
|
2216
|
+
raise exc.InvalidArgumentError(
|
|
2217
|
+
"AUTOSCALE_LIMIT_INVALID",
|
|
2218
|
+
"max_queue_depth must be between 0 and 10000 (0 = unbounded)",
|
|
2219
|
+
)
|
|
2220
|
+
|
|
2221
|
+
|
|
2222
|
+
def _validate_endpoint_autoscale(kind: str, autoscale: Optional[Autoscale]) -> None:
|
|
2223
|
+
"""Reject kind-incompatible policies at decorator time (before any I/O).
|
|
2224
|
+
|
|
2225
|
+
The vLLM preset scrapes the long-lived server's own /metrics, which only
|
|
2226
|
+
exists for ``web_server`` endpoints — mirrors the server-side check.
|
|
2227
|
+
"""
|
|
2228
|
+
if autoscale is None:
|
|
2229
|
+
return
|
|
2230
|
+
if autoscale.vllm_target_waiting is not None and kind != "web_server":
|
|
2231
|
+
raise exc.InvalidArgumentError(
|
|
2232
|
+
"SCALING_POLICY_INVALID",
|
|
2233
|
+
f"vllm_target_waiting requires @app.web_server (got {kind})",
|
|
2234
|
+
)
|
|
2235
|
+
|
|
2236
|
+
|
|
2237
|
+
def _autoscaling_spec(
|
|
2238
|
+
*,
|
|
2239
|
+
min_replicas: Optional[int],
|
|
2240
|
+
max_replicas: Optional[int],
|
|
2241
|
+
max_concurrent_inputs: Optional[int],
|
|
2242
|
+
scale_down_idle_seconds: Optional[int],
|
|
2243
|
+
max_queue_depth: Optional[int] = None,
|
|
2244
|
+
reserved_replicas: Optional[int] = None,
|
|
2245
|
+
) -> dict:
|
|
2246
|
+
_validate_autoscaling(
|
|
2247
|
+
min_replicas=min_replicas,
|
|
2248
|
+
max_replicas=max_replicas,
|
|
2249
|
+
max_concurrent_inputs=max_concurrent_inputs,
|
|
2250
|
+
scale_down_idle_seconds=scale_down_idle_seconds,
|
|
2251
|
+
max_queue_depth=max_queue_depth,
|
|
2252
|
+
)
|
|
2253
|
+
max_eff = 1 if max_replicas is None else max_replicas
|
|
2254
|
+
if reserved_replicas is not None and (
|
|
2255
|
+
not isinstance(reserved_replicas, int)
|
|
2256
|
+
or isinstance(reserved_replicas, bool)
|
|
2257
|
+
or reserved_replicas < 0
|
|
2258
|
+
or reserved_replicas > max_eff
|
|
2259
|
+
):
|
|
2260
|
+
raise exc.InvalidArgumentError(
|
|
2261
|
+
"AUTOSCALE_LIMIT_INVALID",
|
|
2262
|
+
"reserved_replicas must be an integer between 0 and max_replicas",
|
|
2263
|
+
)
|
|
2264
|
+
spec = {
|
|
2265
|
+
"min_replicas": 0 if min_replicas is None else min_replicas,
|
|
2266
|
+
"max_replicas": max_eff,
|
|
2267
|
+
"max_concurrent_inputs": (
|
|
2268
|
+
1 if max_concurrent_inputs is None else max_concurrent_inputs
|
|
2269
|
+
),
|
|
2270
|
+
"scale_down_idle_seconds": (
|
|
2271
|
+
300 if scale_down_idle_seconds is None else scale_down_idle_seconds
|
|
2272
|
+
),
|
|
2273
|
+
}
|
|
2274
|
+
# Sent only when declared: the server default (0 = unbounded) keeps the
|
|
2275
|
+
# historical behavior, and an explicit 0 round-trips to the same thing.
|
|
2276
|
+
if max_queue_depth is not None:
|
|
2277
|
+
spec["max_queue_depth"] = max_queue_depth
|
|
2278
|
+
if reserved_replicas:
|
|
2279
|
+
spec["reserved_replicas"] = reserved_replicas
|
|
2280
|
+
return spec
|
|
2281
|
+
|
|
2282
|
+
|
|
2283
|
+
def _scaling_policy_spec(
|
|
2284
|
+
*,
|
|
2285
|
+
autoscale: Optional[Autoscale],
|
|
2286
|
+
fn_target_inputs: Optional[int],
|
|
2287
|
+
max_concurrent_inputs: int,
|
|
2288
|
+
kind: str,
|
|
2289
|
+
) -> Optional[dict]:
|
|
2290
|
+
"""Build the wire ``scaling_policy`` ({"v": 1, ...}) or ``None`` (v1).
|
|
2291
|
+
|
|
2292
|
+
Merges the endpoint-level ``autoscale=`` declaration with the
|
|
2293
|
+
function-level ``@app.concurrent(target_inputs=)``. Declaring
|
|
2294
|
+
``target_inputs`` in BOTH places is an error (no silent precedence).
|
|
2295
|
+
Mirrors the server-side validation (docs/specs/autoscaling-v2.md §4.3) so
|
|
2296
|
+
a bad policy fails at import/deploy build time, before any network call.
|
|
2297
|
+
"""
|
|
2298
|
+
target = fn_target_inputs
|
|
2299
|
+
buffer_replicas: Optional[int] = None
|
|
2300
|
+
vllm_waiting: Optional[int] = None
|
|
2301
|
+
scaledown_window: Optional[int] = None
|
|
2302
|
+
if autoscale is not None:
|
|
2303
|
+
if autoscale.target_inputs is not None:
|
|
2304
|
+
if target is not None:
|
|
2305
|
+
raise exc.InvalidArgumentError(
|
|
2306
|
+
"SCALING_POLICY_INVALID",
|
|
2307
|
+
"target_inputs declared both in @app.concurrent and in "
|
|
2308
|
+
"bladekit.Autoscale — declare it in exactly one place",
|
|
2309
|
+
)
|
|
2310
|
+
target = autoscale.target_inputs
|
|
2311
|
+
buffer_replicas = autoscale.buffer_replicas
|
|
2312
|
+
vllm_waiting = autoscale.vllm_target_waiting
|
|
2313
|
+
scaledown_window = autoscale.scaledown_window
|
|
2314
|
+
if (
|
|
2315
|
+
target is None
|
|
2316
|
+
and buffer_replicas is None
|
|
2317
|
+
and vllm_waiting is None
|
|
2318
|
+
and scaledown_window is None
|
|
2319
|
+
):
|
|
2320
|
+
return None
|
|
2321
|
+
if target is not None and target > max_concurrent_inputs:
|
|
2322
|
+
raise exc.InvalidArgumentError(
|
|
2323
|
+
"SCALING_POLICY_INVALID",
|
|
2324
|
+
f"target_inputs ({target}) cannot exceed max_concurrent_inputs "
|
|
2325
|
+
f"({max_concurrent_inputs})",
|
|
2326
|
+
)
|
|
2327
|
+
if vllm_waiting is not None and kind != "web_server":
|
|
2328
|
+
raise exc.InvalidArgumentError(
|
|
2329
|
+
"SCALING_POLICY_INVALID",
|
|
2330
|
+
"vllm_target_waiting requires a @app.web_server endpoint "
|
|
2331
|
+
f"(got {kind})",
|
|
2332
|
+
)
|
|
2333
|
+
policy: dict[str, Any] = {"v": 1}
|
|
2334
|
+
if target is not None:
|
|
2335
|
+
policy["target_inputs"] = target
|
|
2336
|
+
if buffer_replicas is not None and buffer_replicas > 0:
|
|
2337
|
+
policy["buffer_replicas"] = buffer_replicas
|
|
2338
|
+
if vllm_waiting is not None:
|
|
2339
|
+
policy["vllm"] = {"target_waiting": vllm_waiting}
|
|
2340
|
+
# Serialized ONLY when the user set it: an older control plane rejects
|
|
2341
|
+
# unknown policy fields with 400 SCALING_POLICY_INVALID.
|
|
2342
|
+
if scaledown_window is not None:
|
|
2343
|
+
policy["scaledown_window_seconds"] = scaledown_window
|
|
2344
|
+
if len(policy) == 1:
|
|
2345
|
+
return None # only {"v":1} left (e.g. buffer_replicas=0): no policy
|
|
2346
|
+
return policy
|
|
2347
|
+
|
|
2348
|
+
|
|
2349
|
+
def _apply_endpoint_marker(target: Any, decl: EndpointDeclaration) -> Any:
|
|
2350
|
+
if isinstance(target, Function):
|
|
2351
|
+
target._add_endpoint(decl)
|
|
2352
|
+
return target
|
|
2353
|
+
existing = list(getattr(target, "__forge_endpoints__", []))
|
|
2354
|
+
existing.append(decl)
|
|
2355
|
+
setattr(target, "__forge_endpoints__", existing)
|
|
2356
|
+
return target
|
|
2357
|
+
|
|
2358
|
+
|
|
2359
|
+
def _apply_concurrency_marker(
|
|
2360
|
+
target: Any, max_concurrent_inputs: int, target_inputs: Optional[int] = None
|
|
2361
|
+
) -> Any:
|
|
2362
|
+
if isinstance(target, Function):
|
|
2363
|
+
target._set_concurrency(max_concurrent_inputs)
|
|
2364
|
+
if target_inputs is not None:
|
|
2365
|
+
target._set_target_inputs(target_inputs)
|
|
2366
|
+
return target
|
|
2367
|
+
setattr(target, "__forge_max_concurrent_inputs__", max_concurrent_inputs)
|
|
2368
|
+
if target_inputs is not None:
|
|
2369
|
+
setattr(target, "__forge_target_inputs__", target_inputs)
|
|
2370
|
+
return target
|
|
2371
|
+
|
|
2372
|
+
|
|
2373
|
+
# Default image when neither the App nor a @app.function specifies one. The
|
|
2374
|
+
# minimal CPU base from the catalog (pre-built, so no build on first run). GPU
|
|
2375
|
+
# work overrides it with a GPU image (e.g. "fabric/torch") + gpu=… per function.
|
|
2376
|
+
# Sent as forge/base until the server knows the fabric/ spelling (to_wire_image).
|
|
2377
|
+
DEFAULT_IMAGE = "fabric/base"
|
|
2378
|
+
|
|
2379
|
+
|
|
2380
|
+
class App:
|
|
2381
|
+
"""An AI Fabric application: a namespace of remotely-callable functions."""
|
|
2382
|
+
|
|
2383
|
+
def __init__(
|
|
2384
|
+
self,
|
|
2385
|
+
name: str,
|
|
2386
|
+
*,
|
|
2387
|
+
image: Any = None,
|
|
2388
|
+
) -> None:
|
|
2389
|
+
self.name = name
|
|
2390
|
+
# Modal-style ergonomics: a function with no image falls back to the
|
|
2391
|
+
# App's default, and the App defaults to the catalog CPU base — so
|
|
2392
|
+
# `@app.function()` just works. Pass image=… to override (any alias,
|
|
2393
|
+
# registry ref, or bladekit.Image DSL).
|
|
2394
|
+
self.default_image = image if image is not None else DEFAULT_IMAGE
|
|
2395
|
+
self.functions: Dict[str, Function] = {}
|
|
2396
|
+
self.classes: Dict[str, Any] = {}
|
|
2397
|
+
self.local_entrypoints: List[Callable[..., Any]] = []
|
|
2398
|
+
# the most-recently registered @app.on_preempt free-function hook
|
|
2399
|
+
# (runner-protocol.md §15.2.1). None until a hook is registered.
|
|
2400
|
+
self.on_preempt_hook: Optional[Callable[..., Any]] = None
|
|
2401
|
+
# cache of definition_hash -> build_id so the same DSL image in a
|
|
2402
|
+
# process resolves its build once.
|
|
2403
|
+
self._image_builds: Dict[str, str] = {}
|
|
2404
|
+
# Bundle root + module mapping, set when the CLI loads an entrypoint file.
|
|
2405
|
+
self._bundle_root: Optional[Path] = None
|
|
2406
|
+
self._entry_module: Optional[str] = None
|
|
2407
|
+
self._client: Any = None
|
|
2408
|
+
self._bundle_id: Optional[str] = None
|
|
2409
|
+
self._bundle_sha256: Optional[str] = None
|
|
2410
|
+
|
|
2411
|
+
def __repr__(self) -> str:
|
|
2412
|
+
return f"<bladekit.App {self.name!r} functions={list(self.functions)}>"
|
|
2413
|
+
|
|
2414
|
+
# -- decorators ---------------------------------------------------------
|
|
2415
|
+
|
|
2416
|
+
def function(
|
|
2417
|
+
self,
|
|
2418
|
+
_fn: Optional[Callable[..., Any]] = None,
|
|
2419
|
+
*,
|
|
2420
|
+
gpu: Optional[str] = None,
|
|
2421
|
+
cpu: Optional[float] = None,
|
|
2422
|
+
memory: Optional[int] = None,
|
|
2423
|
+
timeout: Optional[int] = None,
|
|
2424
|
+
image: Any = None,
|
|
2425
|
+
idle_timeout: Optional[int] = None,
|
|
2426
|
+
volumes: Optional[Dict[str, Any]] = None,
|
|
2427
|
+
secrets: Optional[List[Any]] = None,
|
|
2428
|
+
retries: Optional[dict] = None,
|
|
2429
|
+
priority: Optional[int] = None,
|
|
2430
|
+
namespace: Optional[str] = None,
|
|
2431
|
+
schedule: Any = None,
|
|
2432
|
+
gpu_class: str = "spot",
|
|
2433
|
+
best_effort: bool = False,
|
|
2434
|
+
max_interruptions: Optional[int] = None,
|
|
2435
|
+
):
|
|
2436
|
+
"""Decorator: register a function for remote execution.
|
|
2437
|
+
|
|
2438
|
+
Usable bare (``@app.function``) or with arguments
|
|
2439
|
+
(``@app.function(gpu="RTX-A4500")``). ``image`` is a catalogue alias / raw
|
|
2440
|
+
registry ref (str) OR a ``bladekit.Image`` DSL definition. ``volumes``
|
|
2441
|
+
maps mount paths to ``bladekit.Volume`` objects; ``secrets`` is a list of
|
|
2442
|
+
``bladekit.Secret``.
|
|
2443
|
+
|
|
2444
|
+
Additional options:
|
|
2445
|
+
- ``timeout``: max seconds for ONE execution (server default 3600).
|
|
2446
|
+
It bounds the work, never the worker: a one-shot ``blade run`` /
|
|
2447
|
+
cron / map run is killed past it, while on a DEPLOYED endpoint it
|
|
2448
|
+
caps each request — the service itself never expires (replica
|
|
2449
|
+
lifetime belongs to the autoscaler: ``min_replicas``, idle
|
|
2450
|
+
scale-down, redeploy).
|
|
2451
|
+
- ``retries``: retry policy dict, e.g.
|
|
2452
|
+
``{"max_retries": 3, "backoff": {"initial_seconds": 1, "max_seconds": 60,
|
|
2453
|
+
"factor": 2.0}}``. Applied per-input for ``map()`` calls.
|
|
2454
|
+
- ``priority``: scheduling priority 0–9 (9=urgent, 0=best-effort).
|
|
2455
|
+
Default 5 (FIFO, same as all runs).
|
|
2456
|
+
- ``namespace``: team namespace for GPU quota accounting.
|
|
2457
|
+
- ``schedule``: a ``bladekit.Cron(...)`` object for cron-scheduled execution
|
|
2458
|
+
(only takes effect in a deployed app, not in ``blade run``).
|
|
2459
|
+
- ``gpu="RTX-A4500:4"``: multi-GPU per container (``":N"`` suffix
|
|
2460
|
+
sets ``gpu_count``; N must be 1–8).
|
|
2461
|
+
"""
|
|
2462
|
+
# Validate priority/retries params at decoration time (not at call time).
|
|
2463
|
+
# retries accepts an int shorthand (retries=3) or a RetryPolicy dict.
|
|
2464
|
+
if priority is not None:
|
|
2465
|
+
_validate_priority(priority)
|
|
2466
|
+
if retries is not None:
|
|
2467
|
+
retries = _normalize_retries(retries)
|
|
2468
|
+
# validate gpu_class client-side with a clear error.
|
|
2469
|
+
_validate_gpu_class(gpu_class)
|
|
2470
|
+
|
|
2471
|
+
def wrap(fn: Callable[..., Any]) -> Function:
|
|
2472
|
+
if isinstance(fn, Function):
|
|
2473
|
+
if fn._app is not self:
|
|
2474
|
+
raise exc.InvalidArgumentError(
|
|
2475
|
+
"INVALID_ARGUMENT",
|
|
2476
|
+
"@app.function cannot wrap a Function from another App",
|
|
2477
|
+
)
|
|
2478
|
+
if gpu is not None:
|
|
2479
|
+
fn.gpu = gpu
|
|
2480
|
+
if cpu is not None:
|
|
2481
|
+
fn.cpu = cpu
|
|
2482
|
+
if memory is not None:
|
|
2483
|
+
fn.memory = memory
|
|
2484
|
+
if timeout is not None:
|
|
2485
|
+
fn.timeout = timeout
|
|
2486
|
+
if image is not None:
|
|
2487
|
+
fn.image = image
|
|
2488
|
+
if idle_timeout is not None:
|
|
2489
|
+
fn.idle_timeout = idle_timeout
|
|
2490
|
+
if volumes is not None:
|
|
2491
|
+
fn.volumes = volumes
|
|
2492
|
+
if secrets is not None:
|
|
2493
|
+
fn.secrets = secrets
|
|
2494
|
+
if retries is not None:
|
|
2495
|
+
fn.retries = retries
|
|
2496
|
+
if priority is not None:
|
|
2497
|
+
fn.priority = priority
|
|
2498
|
+
if namespace is not None:
|
|
2499
|
+
fn.namespace = namespace
|
|
2500
|
+
if schedule is not None:
|
|
2501
|
+
fn.schedule = schedule
|
|
2502
|
+
# only override on a non-default (gpu_class can't be None,
|
|
2503
|
+
# so a bare default must not clobber a value set on the wrapped fn).
|
|
2504
|
+
if gpu_class != "spot":
|
|
2505
|
+
fn.gpu_class = gpu_class
|
|
2506
|
+
if best_effort:
|
|
2507
|
+
fn.best_effort = best_effort
|
|
2508
|
+
if max_interruptions is not None:
|
|
2509
|
+
fn.max_interruptions = max_interruptions
|
|
2510
|
+
self.functions[fn.__name__] = fn
|
|
2511
|
+
return fn
|
|
2512
|
+
f = Function(
|
|
2513
|
+
self,
|
|
2514
|
+
fn,
|
|
2515
|
+
gpu=gpu,
|
|
2516
|
+
cpu=cpu,
|
|
2517
|
+
memory=memory,
|
|
2518
|
+
timeout=timeout,
|
|
2519
|
+
image=image,
|
|
2520
|
+
idle_timeout=idle_timeout,
|
|
2521
|
+
volumes=volumes,
|
|
2522
|
+
secrets=secrets,
|
|
2523
|
+
retries=retries,
|
|
2524
|
+
priority=priority,
|
|
2525
|
+
namespace=namespace,
|
|
2526
|
+
schedule=schedule,
|
|
2527
|
+
gpu_class=gpu_class,
|
|
2528
|
+
best_effort=best_effort,
|
|
2529
|
+
max_interruptions=max_interruptions,
|
|
2530
|
+
)
|
|
2531
|
+
self.functions[f.__name__] = f
|
|
2532
|
+
return f
|
|
2533
|
+
|
|
2534
|
+
if _fn is not None:
|
|
2535
|
+
return wrap(_fn)
|
|
2536
|
+
return wrap
|
|
2537
|
+
|
|
2538
|
+
def web_endpoint(
|
|
2539
|
+
self,
|
|
2540
|
+
_fn: Optional[Callable[..., Any]] = None,
|
|
2541
|
+
*,
|
|
2542
|
+
path: Optional[str] = None,
|
|
2543
|
+
methods: Optional[Sequence[str]] = None,
|
|
2544
|
+
name: Optional[str] = None,
|
|
2545
|
+
min_replicas: Optional[int] = None,
|
|
2546
|
+
max_replicas: Optional[int] = None,
|
|
2547
|
+
max_concurrent_inputs: Optional[int] = None,
|
|
2548
|
+
scale_down_idle_seconds: Optional[int] = None,
|
|
2549
|
+
max_queue_depth: Optional[int] = None,
|
|
2550
|
+
reserved_replicas: Optional[int] = None,
|
|
2551
|
+
public: bool = False,
|
|
2552
|
+
autoscale: Optional[Autoscale] = None,
|
|
2553
|
+
):
|
|
2554
|
+
"""Decorator: expose a function through HTTP call routing.
|
|
2555
|
+
|
|
2556
|
+
Usable alone (it registers a function with default resources) or stacked
|
|
2557
|
+
with ``@app.function`` / ``@app.concurrent`` in either order.
|
|
2558
|
+
|
|
2559
|
+
Endpoints are PRIVATE by default (the /serve ingress requires a valid
|
|
2560
|
+
user token). Pass ``public=True`` to opt out (no auth).
|
|
2561
|
+
|
|
2562
|
+
``max_queue_depth`` (0..10000) bounds how many requests may wait for a
|
|
2563
|
+
replica slot at the platform ingress; at the bound, new requests get an
|
|
2564
|
+
immediate ``503 ENDPOINT_QUEUE_FULL`` + ``Retry-After`` instead of
|
|
2565
|
+
waiting out a cold start. Unset/0 = unbounded (the historical
|
|
2566
|
+
behavior).
|
|
2567
|
+
"""
|
|
2568
|
+
methods_tuple = _normalize_methods(methods, default=("POST",))
|
|
2569
|
+
_validate_autoscaling(
|
|
2570
|
+
min_replicas=min_replicas,
|
|
2571
|
+
max_replicas=max_replicas,
|
|
2572
|
+
max_concurrent_inputs=max_concurrent_inputs,
|
|
2573
|
+
scale_down_idle_seconds=scale_down_idle_seconds,
|
|
2574
|
+
max_queue_depth=max_queue_depth,
|
|
2575
|
+
)
|
|
2576
|
+
_validate_endpoint_autoscale("web_endpoint", autoscale)
|
|
2577
|
+
|
|
2578
|
+
def wrap(fn: Callable[..., Any]) -> Function:
|
|
2579
|
+
endpoint_path = _validate_path(path or f"/{getattr(fn, '__name__', 'endpoint')}")
|
|
2580
|
+
decl = EndpointDeclaration(
|
|
2581
|
+
name=name,
|
|
2582
|
+
kind="web_endpoint",
|
|
2583
|
+
path=endpoint_path,
|
|
2584
|
+
methods=methods_tuple,
|
|
2585
|
+
min_replicas=min_replicas,
|
|
2586
|
+
max_replicas=max_replicas,
|
|
2587
|
+
max_concurrent_inputs=max_concurrent_inputs,
|
|
2588
|
+
scale_down_idle_seconds=scale_down_idle_seconds,
|
|
2589
|
+
max_queue_depth=max_queue_depth,
|
|
2590
|
+
reserved_replicas=reserved_replicas,
|
|
2591
|
+
public=public,
|
|
2592
|
+
autoscale=autoscale,
|
|
2593
|
+
)
|
|
2594
|
+
if isinstance(fn, Function):
|
|
2595
|
+
return _apply_endpoint_marker(fn, decl)
|
|
2596
|
+
_apply_endpoint_marker(fn, decl)
|
|
2597
|
+
return self.function(fn)
|
|
2598
|
+
|
|
2599
|
+
if _fn is not None:
|
|
2600
|
+
return wrap(_fn)
|
|
2601
|
+
return wrap
|
|
2602
|
+
|
|
2603
|
+
def web_server(
|
|
2604
|
+
self,
|
|
2605
|
+
_fn: Optional[Callable[..., Any]] = None,
|
|
2606
|
+
*,
|
|
2607
|
+
port: Optional[int] = None,
|
|
2608
|
+
startup_timeout: int = 300,
|
|
2609
|
+
path: str = "/",
|
|
2610
|
+
name: Optional[str] = None,
|
|
2611
|
+
min_replicas: Optional[int] = None,
|
|
2612
|
+
max_replicas: Optional[int] = None,
|
|
2613
|
+
max_concurrent_inputs: Optional[int] = None,
|
|
2614
|
+
scale_down_idle_seconds: Optional[int] = None,
|
|
2615
|
+
max_queue_depth: Optional[int] = None,
|
|
2616
|
+
reserved_replicas: Optional[int] = None,
|
|
2617
|
+
public: bool = False,
|
|
2618
|
+
autoscale: Optional[Autoscale] = None,
|
|
2619
|
+
snapshot: bool = False,
|
|
2620
|
+
):
|
|
2621
|
+
"""Decorator: serve a long-lived HTTP server running inside the container.
|
|
2622
|
+
|
|
2623
|
+
Unlike ``web_endpoint`` (a handler called per request), the decorated
|
|
2624
|
+
function is invoked ONCE at container boot to start a long-lived server
|
|
2625
|
+
— typically a non-blocking ``subprocess.Popen(["vllm", "serve", …])``.
|
|
2626
|
+
AI Fabric probes the loopback until the server accepts connections (up to
|
|
2627
|
+
``startup_timeout`` seconds) then forwards every web request to it
|
|
2628
|
+
(runner-protocol.md §13.5). The server owns its own method/path routing,
|
|
2629
|
+
so the endpoint captures all methods under ``path``.
|
|
2630
|
+
|
|
2631
|
+
**Port.** Each container is assigned a node-unique loopback port via the
|
|
2632
|
+
``FABRIC_SERVER_PORT`` env var, so co-located replicas never collide on a
|
|
2633
|
+
shared host port. Your server MUST bind ``$FABRIC_SERVER_PORT``. Passing a
|
|
2634
|
+
fixed ``port=`` is rejected (it re-introduced co-location cross-talk).
|
|
2635
|
+
|
|
2636
|
+
Combine with ``@app.concurrent(max_inputs=N)`` to let multiple requests
|
|
2637
|
+
be forwarded concurrently — this is what enables server-side continuous
|
|
2638
|
+
batching (e.g. vLLM). Stackable with ``@app.function`` / ``@app.concurrent``
|
|
2639
|
+
in either order. Only non-streaming responses are supported for now
|
|
2640
|
+
(SSE/WebSocket deferred, §13.5).
|
|
2641
|
+
|
|
2642
|
+
**Snapshot.** ``snapshot=True`` (coldstart-snapshot.md §5) opts the
|
|
2643
|
+
endpoint into GPU checkpoint/restore: the platform injects the
|
|
2644
|
+
snapshotability recipe into every replica, checkpoints a warm replica at
|
|
2645
|
+
drain and restores it on scale-up (falling back to a normal boot on any
|
|
2646
|
+
error). Valid ONLY on a single-GPU ``web_server`` (``gpu_count == 1``);
|
|
2647
|
+
otherwise it raises ``bladekit.InvalidArgumentError`` at deploy build time.
|
|
2648
|
+
|
|
2649
|
+
**Queue bound.** ``max_queue_depth`` (0..10000) caps how many requests
|
|
2650
|
+
may wait for a replica slot at the platform ingress — mostly the
|
|
2651
|
+
cold-start window; at the bound new requests get an immediate
|
|
2652
|
+
``503 ENDPOINT_QUEUE_FULL`` + ``Retry-After``. Unset/0 = unbounded
|
|
2653
|
+
(the historical behavior). Note the scope: traffic load-balanced
|
|
2654
|
+
straight to a WARM replica is not queued by the platform — its backlog
|
|
2655
|
+
lives inside your own server (e.g. the vLLM scheduler queue).
|
|
2656
|
+
|
|
2657
|
+
Example::
|
|
2658
|
+
|
|
2659
|
+
@app.web_server(startup_timeout=600)
|
|
2660
|
+
@app.concurrent(max_inputs=32)
|
|
2661
|
+
@app.function(gpu="RTX-A4500", image=vllm_image)
|
|
2662
|
+
def serve():
|
|
2663
|
+
import os, subprocess
|
|
2664
|
+
port = os.environ["FABRIC_SERVER_PORT"]
|
|
2665
|
+
subprocess.Popen(
|
|
2666
|
+
["vllm", "serve", "meta-llama/Llama-3.1-8B-Instruct",
|
|
2667
|
+
"--port", port],
|
|
2668
|
+
)
|
|
2669
|
+
"""
|
|
2670
|
+
_reject_pinned_server_port(port)
|
|
2671
|
+
_validate_startup_timeout(startup_timeout)
|
|
2672
|
+
endpoint_path = _validate_path(path)
|
|
2673
|
+
_validate_autoscaling(
|
|
2674
|
+
min_replicas=min_replicas,
|
|
2675
|
+
max_replicas=max_replicas,
|
|
2676
|
+
max_concurrent_inputs=max_concurrent_inputs,
|
|
2677
|
+
scale_down_idle_seconds=scale_down_idle_seconds,
|
|
2678
|
+
max_queue_depth=max_queue_depth,
|
|
2679
|
+
)
|
|
2680
|
+
_validate_endpoint_autoscale("web_server", autoscale)
|
|
2681
|
+
|
|
2682
|
+
def wrap(fn: Callable[..., Any]) -> Function:
|
|
2683
|
+
decl = EndpointDeclaration(
|
|
2684
|
+
name=name,
|
|
2685
|
+
kind="web_server",
|
|
2686
|
+
path=endpoint_path,
|
|
2687
|
+
methods=_ASGI_METHODS, # the server owns method routing
|
|
2688
|
+
min_replicas=min_replicas,
|
|
2689
|
+
max_replicas=max_replicas,
|
|
2690
|
+
max_concurrent_inputs=max_concurrent_inputs,
|
|
2691
|
+
scale_down_idle_seconds=scale_down_idle_seconds,
|
|
2692
|
+
max_queue_depth=max_queue_depth,
|
|
2693
|
+
reserved_replicas=reserved_replicas,
|
|
2694
|
+
public=public,
|
|
2695
|
+
port=port,
|
|
2696
|
+
startup_timeout_seconds=startup_timeout,
|
|
2697
|
+
autoscale=autoscale,
|
|
2698
|
+
snapshot=snapshot,
|
|
2699
|
+
)
|
|
2700
|
+
if isinstance(fn, Function):
|
|
2701
|
+
return _apply_endpoint_marker(fn, decl)
|
|
2702
|
+
_apply_endpoint_marker(fn, decl)
|
|
2703
|
+
return self.function(fn)
|
|
2704
|
+
|
|
2705
|
+
if _fn is not None:
|
|
2706
|
+
return wrap(_fn)
|
|
2707
|
+
return wrap
|
|
2708
|
+
|
|
2709
|
+
def asgi_app(
|
|
2710
|
+
self,
|
|
2711
|
+
_fn: Optional[Callable[..., Any]] = None,
|
|
2712
|
+
*,
|
|
2713
|
+
path: str = "/",
|
|
2714
|
+
startup_timeout: int = 300,
|
|
2715
|
+
name: Optional[str] = None,
|
|
2716
|
+
min_replicas: Optional[int] = None,
|
|
2717
|
+
max_replicas: Optional[int] = None,
|
|
2718
|
+
max_concurrent_inputs: Optional[int] = None,
|
|
2719
|
+
scale_down_idle_seconds: Optional[int] = None,
|
|
2720
|
+
max_queue_depth: Optional[int] = None,
|
|
2721
|
+
reserved_replicas: Optional[int] = None,
|
|
2722
|
+
public: bool = False,
|
|
2723
|
+
autoscale: Optional[Autoscale] = None,
|
|
2724
|
+
):
|
|
2725
|
+
"""Decorator: serve an ASGI app (FastAPI, Starlette, …) as an endpoint.
|
|
2726
|
+
|
|
2727
|
+
The decorated function returns the ASGI app object. At boot the runner
|
|
2728
|
+
allocates a free loopback port, starts ``uvicorn`` on the returned app,
|
|
2729
|
+
and forwards each web request to it (runner-protocol.md §13.5). ``uvicorn``
|
|
2730
|
+
must be present in the image. ``startup_timeout`` (seconds) bounds how long
|
|
2731
|
+
the runner probes the loopback before the replica is marked ready; the
|
|
2732
|
+
replica fails if the server never answers.
|
|
2733
|
+
|
|
2734
|
+
Stackable with ``@app.function`` / ``@app.concurrent`` like ``web_endpoint``.
|
|
2735
|
+
Only non-streaming responses are supported for now — SSE/WebSocket are
|
|
2736
|
+
deferred (§13.5).
|
|
2737
|
+
"""
|
|
2738
|
+
endpoint_path = _validate_path(path)
|
|
2739
|
+
_validate_startup_timeout(startup_timeout)
|
|
2740
|
+
_validate_autoscaling(
|
|
2741
|
+
min_replicas=min_replicas,
|
|
2742
|
+
max_replicas=max_replicas,
|
|
2743
|
+
max_concurrent_inputs=max_concurrent_inputs,
|
|
2744
|
+
scale_down_idle_seconds=scale_down_idle_seconds,
|
|
2745
|
+
max_queue_depth=max_queue_depth,
|
|
2746
|
+
)
|
|
2747
|
+
_validate_endpoint_autoscale("asgi_app", autoscale)
|
|
2748
|
+
|
|
2749
|
+
def wrap(fn: Callable[..., Any]) -> Function:
|
|
2750
|
+
decl = EndpointDeclaration(
|
|
2751
|
+
name=name,
|
|
2752
|
+
kind="asgi_app",
|
|
2753
|
+
path=endpoint_path,
|
|
2754
|
+
methods=_ASGI_METHODS,
|
|
2755
|
+
min_replicas=min_replicas,
|
|
2756
|
+
max_replicas=max_replicas,
|
|
2757
|
+
max_concurrent_inputs=max_concurrent_inputs,
|
|
2758
|
+
scale_down_idle_seconds=scale_down_idle_seconds,
|
|
2759
|
+
max_queue_depth=max_queue_depth,
|
|
2760
|
+
reserved_replicas=reserved_replicas,
|
|
2761
|
+
autoscale=autoscale,
|
|
2762
|
+
# Deployed endpoints are PRIVATE by default (a valid user token is
|
|
2763
|
+
# required). public=True opts out so the URL opens directly in a
|
|
2764
|
+
# browser — needed for ASGI apps that serve a UI / WebRTC signaling.
|
|
2765
|
+
public=public,
|
|
2766
|
+
# Runner allocates the loopback port for uvicorn; server_port stays None.
|
|
2767
|
+
port=None,
|
|
2768
|
+
startup_timeout_seconds=startup_timeout,
|
|
2769
|
+
)
|
|
2770
|
+
if isinstance(fn, Function):
|
|
2771
|
+
return _apply_endpoint_marker(fn, decl)
|
|
2772
|
+
_apply_endpoint_marker(fn, decl)
|
|
2773
|
+
return self.function(fn)
|
|
2774
|
+
|
|
2775
|
+
if _fn is not None:
|
|
2776
|
+
return wrap(_fn)
|
|
2777
|
+
return wrap
|
|
2778
|
+
|
|
2779
|
+
def concurrent(
|
|
2780
|
+
self,
|
|
2781
|
+
_fn: Optional[Callable[..., Any]] = None,
|
|
2782
|
+
*,
|
|
2783
|
+
max_inputs: Optional[int] = None,
|
|
2784
|
+
max_concurrent_inputs: Optional[int] = None,
|
|
2785
|
+
target_inputs: Optional[int] = None,
|
|
2786
|
+
):
|
|
2787
|
+
"""Decorator: set per-replica HTTP concurrency for endpoints.
|
|
2788
|
+
|
|
2789
|
+
``max_inputs`` is the admission ceiling per replica. The optional
|
|
2790
|
+
``target_inputs`` is what the autoscaler AIMS for (autoscaling v2,
|
|
2791
|
+
Modal semantics): the fleet is sized to hold ~``target_inputs``
|
|
2792
|
+
in-flight calls per replica, and the margin up to ``max_inputs``
|
|
2793
|
+
absorbs bursts while new replicas boot. Pick ``target_inputs`` from
|
|
2794
|
+
your latency budget and ``max_inputs`` from resource limits::
|
|
2795
|
+
|
|
2796
|
+
@app.concurrent(max_inputs=32, target_inputs=24)
|
|
2797
|
+
"""
|
|
2798
|
+
value = max_concurrent_inputs if max_concurrent_inputs is not None else max_inputs
|
|
2799
|
+
if value is None:
|
|
2800
|
+
raise exc.InvalidArgumentError(
|
|
2801
|
+
"INVALID_ARGUMENT",
|
|
2802
|
+
"app.concurrent requires max_inputs= or max_concurrent_inputs=",
|
|
2803
|
+
)
|
|
2804
|
+
_validate_concurrency(value)
|
|
2805
|
+
if target_inputs is not None:
|
|
2806
|
+
if (
|
|
2807
|
+
not isinstance(target_inputs, int)
|
|
2808
|
+
or isinstance(target_inputs, bool)
|
|
2809
|
+
or target_inputs < 1
|
|
2810
|
+
or target_inputs > 100
|
|
2811
|
+
):
|
|
2812
|
+
raise exc.InvalidArgumentError(
|
|
2813
|
+
"SCALING_POLICY_INVALID",
|
|
2814
|
+
"target_inputs must be an integer between 1 and 100",
|
|
2815
|
+
)
|
|
2816
|
+
if target_inputs > value:
|
|
2817
|
+
raise exc.InvalidArgumentError(
|
|
2818
|
+
"SCALING_POLICY_INVALID",
|
|
2819
|
+
f"target_inputs ({target_inputs}) cannot exceed "
|
|
2820
|
+
f"max_inputs ({value})",
|
|
2821
|
+
)
|
|
2822
|
+
|
|
2823
|
+
def wrap(fn: Callable[..., Any]) -> Any:
|
|
2824
|
+
return _apply_concurrency_marker(fn, value, target_inputs)
|
|
2825
|
+
|
|
2826
|
+
if _fn is not None:
|
|
2827
|
+
return wrap(_fn)
|
|
2828
|
+
return wrap
|
|
2829
|
+
|
|
2830
|
+
# -- @app.cls / @app.method ----------------------------------------
|
|
2831
|
+
|
|
2832
|
+
def cls(
|
|
2833
|
+
self,
|
|
2834
|
+
_cls: Optional[type] = None,
|
|
2835
|
+
*,
|
|
2836
|
+
gpu: Optional[str] = None,
|
|
2837
|
+
cpu: Optional[float] = None,
|
|
2838
|
+
memory: Optional[int] = None,
|
|
2839
|
+
timeout: Optional[int] = None,
|
|
2840
|
+
image: Any = None,
|
|
2841
|
+
idle_timeout: Optional[int] = None,
|
|
2842
|
+
volumes: Optional[Dict[str, Any]] = None,
|
|
2843
|
+
secrets: Optional[List[Any]] = None,
|
|
2844
|
+
gpu_class: str = "spot",
|
|
2845
|
+
best_effort: bool = False,
|
|
2846
|
+
max_interruptions: Optional[int] = None,
|
|
2847
|
+
):
|
|
2848
|
+
"""Decorator: register a class whose ``@app.method``s are callable remotely.
|
|
2849
|
+
|
|
2850
|
+
Methods marked ``@bladekit.enter()`` run once per container (model load).
|
|
2851
|
+
The runner instantiates the class with no args, runs the enter hooks,
|
|
2852
|
+
then calls the target method (runner-protocol.md §10). A method marked
|
|
2853
|
+
``@bladekit.on_preempt`` runs on the preemption grace window
|
|
2854
|
+
(runner-protocol.md §15.2.1).
|
|
2855
|
+
|
|
2856
|
+
``timeout`` has the same semantics as ``@app.function``: max seconds
|
|
2857
|
+
for one call (or one request on a deployed endpoint) — never the
|
|
2858
|
+
lifetime of a deployed service, which lives under the autoscaler.
|
|
2859
|
+
"""
|
|
2860
|
+
from ._cls import Cls
|
|
2861
|
+
|
|
2862
|
+
_validate_gpu_class(gpu_class)
|
|
2863
|
+
|
|
2864
|
+
def wrap(user_cls: type):
|
|
2865
|
+
c = Cls(
|
|
2866
|
+
self,
|
|
2867
|
+
user_cls,
|
|
2868
|
+
gpu=gpu,
|
|
2869
|
+
cpu=cpu,
|
|
2870
|
+
memory=memory,
|
|
2871
|
+
timeout=timeout,
|
|
2872
|
+
image=image,
|
|
2873
|
+
idle_timeout=idle_timeout,
|
|
2874
|
+
volumes=volumes,
|
|
2875
|
+
secrets=secrets,
|
|
2876
|
+
gpu_class=gpu_class,
|
|
2877
|
+
best_effort=best_effort,
|
|
2878
|
+
max_interruptions=max_interruptions,
|
|
2879
|
+
)
|
|
2880
|
+
self.classes[c.__name__] = c
|
|
2881
|
+
# A @app.web_endpoint / @app.asgi_app / @app.web_server applied to a
|
|
2882
|
+
# METHOD of this class registered a module-level Function wrapping the
|
|
2883
|
+
# UNBOUND method with no class_name — so the deploy manifest omitted
|
|
2884
|
+
# class_name, the runner took the plain-function path, and every call
|
|
2885
|
+
# failed with "'Cls' object has no attribute '<method>'". Rewire each
|
|
2886
|
+
# such method: (1) tag the Function with this class so the manifest
|
|
2887
|
+
# carries class_name and the runner takes the instantiate-then-call
|
|
2888
|
+
# path (runner-protocol §10, exactly like @app.method); (2) restore
|
|
2889
|
+
# the raw method on the class so the runner's getattr(instance, name)
|
|
2890
|
+
# binds it. Without (2) the class attribute would still be the
|
|
2891
|
+
# Function object, not a bound method.
|
|
2892
|
+
for attr_name, attr in list(vars(user_cls).items()):
|
|
2893
|
+
if isinstance(attr, Function) and attr._endpoints:
|
|
2894
|
+
attr.class_name = c.__name__
|
|
2895
|
+
# web_endpoint built this Function (via self.function(fn))
|
|
2896
|
+
# BEFORE @app.cls ran, so it has no image/resources — it would
|
|
2897
|
+
# deploy on forge/base with no GPU (the method's calls then 500
|
|
2898
|
+
# with ModuleNotFoundError: torch). Inherit the @app.cls config
|
|
2899
|
+
# for anything the method didn't set itself, exactly as
|
|
2900
|
+
# @app.method methods inherit it via Cls._function_for.
|
|
2901
|
+
if attr.image is None:
|
|
2902
|
+
attr.image = image
|
|
2903
|
+
if attr.gpu is None:
|
|
2904
|
+
attr.gpu = gpu
|
|
2905
|
+
if attr.cpu is None:
|
|
2906
|
+
attr.cpu = cpu
|
|
2907
|
+
if attr.memory is None:
|
|
2908
|
+
attr.memory = memory
|
|
2909
|
+
if attr.timeout is None:
|
|
2910
|
+
attr.timeout = timeout
|
|
2911
|
+
if attr.idle_timeout is None:
|
|
2912
|
+
attr.idle_timeout = idle_timeout
|
|
2913
|
+
if attr.volumes is None:
|
|
2914
|
+
attr.volumes = volumes
|
|
2915
|
+
if attr.secrets is None:
|
|
2916
|
+
attr.secrets = secrets
|
|
2917
|
+
if attr.gpu_class == "spot":
|
|
2918
|
+
attr.gpu_class = gpu_class
|
|
2919
|
+
if not attr.best_effort:
|
|
2920
|
+
attr.best_effort = best_effort
|
|
2921
|
+
if attr.max_interruptions is None:
|
|
2922
|
+
attr.max_interruptions = max_interruptions
|
|
2923
|
+
setattr(user_cls, attr_name, attr._fn)
|
|
2924
|
+
return c
|
|
2925
|
+
|
|
2926
|
+
if _cls is not None:
|
|
2927
|
+
return wrap(_cls)
|
|
2928
|
+
return wrap
|
|
2929
|
+
|
|
2930
|
+
def method(self, _fn: Optional[Callable[..., Any]] = None):
|
|
2931
|
+
"""Decorator: mark a method of an ``@app.cls`` class as remotely callable."""
|
|
2932
|
+
from ._cls import Method
|
|
2933
|
+
|
|
2934
|
+
def wrap(fn: Callable[..., Any]) -> "Method":
|
|
2935
|
+
return Method(fn)
|
|
2936
|
+
|
|
2937
|
+
if _fn is not None:
|
|
2938
|
+
return wrap(_fn)
|
|
2939
|
+
return wrap
|
|
2940
|
+
|
|
2941
|
+
# -- preemption hook ---------------------------------------------
|
|
2942
|
+
|
|
2943
|
+
def on_preempt(self, fn: Callable[..., Any]) -> Callable[..., Any]:
|
|
2944
|
+
"""Decorator: register a callback to run when the container is preempted.
|
|
2945
|
+
|
|
2946
|
+
Usable as ``@app.on_preempt`` (alias of the module-level
|
|
2947
|
+
``@bladekit.on_preempt``). Marks the callback with the SDK↔runner
|
|
2948
|
+
``__forge_on_preempt__`` attribute so the runner discovers it on the
|
|
2949
|
+
SIGTERM grace window the same way it finds ``@bladekit.enter`` hooks
|
|
2950
|
+
(runner-protocol.md §15.2.1). Works on a free function (found in the
|
|
2951
|
+
bundle module) or an ``@app.cls`` method (found on the live instance).
|
|
2952
|
+
The last registered callback wins (deterministic, one callback).
|
|
2953
|
+
"""
|
|
2954
|
+
from ._preempt import on_preempt as _mark
|
|
2955
|
+
|
|
2956
|
+
marked = _mark(fn)
|
|
2957
|
+
# Track the most-recently registered free-function hook on the app too —
|
|
2958
|
+
# convenient for introspection/tests; the runner's source of truth is the
|
|
2959
|
+
# __forge_on_preempt__ marker it scans for, not this attribute.
|
|
2960
|
+
self.on_preempt_hook = marked
|
|
2961
|
+
return marked
|
|
2962
|
+
|
|
2963
|
+
# -- image builds --------------------------------------------------
|
|
2964
|
+
|
|
2965
|
+
def _ensure_image_build(self, image: Any) -> str:
|
|
2966
|
+
"""Create-or-find the build for a ``bladekit.Image`` and wait for success.
|
|
2967
|
+
|
|
2968
|
+
Idempotent by definition hash (image-dsl.md §7): the server dedups, so
|
|
2969
|
+
re-running an unchanged image triggers zero rebuild. If the build is not
|
|
2970
|
+
yet ``succeeded`` the build logs are streamed live; a failed build raises
|
|
2971
|
+
an actionable :class:`bladekit.ImageBuildError`.
|
|
2972
|
+
"""
|
|
2973
|
+
definition = image.to_definition()
|
|
2974
|
+
definition_hash = image.definition_hash()
|
|
2975
|
+
cache = self._image_builds
|
|
2976
|
+
cached = cache.get(definition_hash)
|
|
2977
|
+
if cached is not None:
|
|
2978
|
+
return cached
|
|
2979
|
+
|
|
2980
|
+
client = self._require_client()
|
|
2981
|
+
from ._build import resolve_image_build
|
|
2982
|
+
|
|
2983
|
+
build_id = resolve_image_build(
|
|
2984
|
+
client,
|
|
2985
|
+
definition,
|
|
2986
|
+
definition_hash,
|
|
2987
|
+
context_data=getattr(image, "_context_data", None),
|
|
2988
|
+
)
|
|
2989
|
+
cache[definition_hash] = build_id
|
|
2990
|
+
return build_id
|
|
2991
|
+
|
|
2992
|
+
# -- deployment manifests -----------------------------------------
|
|
2993
|
+
|
|
2994
|
+
def deployment_manifest(
|
|
2995
|
+
self,
|
|
2996
|
+
*,
|
|
2997
|
+
bundle_id: Optional[str] = None,
|
|
2998
|
+
labels: Optional[dict[str, str]] = None,
|
|
2999
|
+
) -> DeploymentManifest:
|
|
3000
|
+
"""Return a serializable manifest without uploading/building anything."""
|
|
3001
|
+
return self._deployment_manifest(bundle_id=bundle_id, labels=labels, resolve=False)
|
|
3002
|
+
|
|
3003
|
+
def _deployment_manifest(
|
|
3004
|
+
self,
|
|
3005
|
+
*,
|
|
3006
|
+
bundle_id: Optional[str] = None,
|
|
3007
|
+
labels: Optional[dict[str, str]] = None,
|
|
3008
|
+
resolve: bool = False,
|
|
3009
|
+
require_endpoints: bool = False,
|
|
3010
|
+
) -> DeploymentManifest:
|
|
3011
|
+
from ._serialize import cloudpickle_version
|
|
3012
|
+
|
|
3013
|
+
if resolve and bundle_id is None:
|
|
3014
|
+
bundle_id = self._ensure_bundle()
|
|
3015
|
+
|
|
3016
|
+
functions = [fn._manifest_function() for fn in self.functions.values()]
|
|
3017
|
+
classes = []
|
|
3018
|
+
for cls in self.classes.values():
|
|
3019
|
+
classes.append(
|
|
3020
|
+
{
|
|
3021
|
+
"name": cls.__name__,
|
|
3022
|
+
"qualname": cls.__qualname__,
|
|
3023
|
+
"methods": sorted(cls.methods.keys()),
|
|
3024
|
+
}
|
|
3025
|
+
)
|
|
3026
|
+
|
|
3027
|
+
endpoints: list[dict] = []
|
|
3028
|
+
endpoint_functions: list[Function] = []
|
|
3029
|
+
seen_routes: set[tuple[str, str]] = set()
|
|
3030
|
+
for fn in self.functions.values():
|
|
3031
|
+
for spec in fn._endpoint_specs():
|
|
3032
|
+
for method in spec["methods"]:
|
|
3033
|
+
route = (method, spec["path"])
|
|
3034
|
+
if route in seen_routes:
|
|
3035
|
+
raise exc.ConflictError(
|
|
3036
|
+
"ENDPOINT_ROUTE_CONFLICT",
|
|
3037
|
+
f"duplicate endpoint route {method} {spec['path']}",
|
|
3038
|
+
)
|
|
3039
|
+
seen_routes.add(route)
|
|
3040
|
+
endpoints.append(spec)
|
|
3041
|
+
endpoint_functions.append(fn)
|
|
3042
|
+
|
|
3043
|
+
crons: list[dict] = []
|
|
3044
|
+
cron_functions: list[Function] = []
|
|
3045
|
+
for fn in self.functions.values():
|
|
3046
|
+
cron_spec = fn._cron_spec()
|
|
3047
|
+
if cron_spec is not None:
|
|
3048
|
+
crons.append(cron_spec)
|
|
3049
|
+
cron_functions.append(fn)
|
|
3050
|
+
|
|
3051
|
+
if len(endpoints) > 64:
|
|
3052
|
+
raise exc.InvalidArgumentError(
|
|
3053
|
+
"INVALID_ARGUMENT", "deployments are limited to 64 endpoints"
|
|
3054
|
+
)
|
|
3055
|
+
if len(crons) > 64:
|
|
3056
|
+
raise exc.InvalidArgumentError(
|
|
3057
|
+
"INVALID_ARGUMENT", "deployments are limited to 64 cron schedules"
|
|
3058
|
+
)
|
|
3059
|
+
# a deployment may declare endpoints OR crons (or both). blade deploy
|
|
3060
|
+
# only needs at least one of the two to be a valid deployment.
|
|
3061
|
+
if require_endpoints and not endpoints and not crons:
|
|
3062
|
+
raise exc.InvalidArgumentError(
|
|
3063
|
+
"INVALID_ARGUMENT",
|
|
3064
|
+
"blade deploy requires at least one @app.web_endpoint(), "
|
|
3065
|
+
"@app.web_server(), @app.asgi_app(), or "
|
|
3066
|
+
"@app.function(schedule=bladekit.Cron(...))",
|
|
3067
|
+
)
|
|
3068
|
+
|
|
3069
|
+
image: Optional[str] = None
|
|
3070
|
+
image_build_id: Optional[str] = None
|
|
3071
|
+
image_functions = endpoint_functions + cron_functions
|
|
3072
|
+
if image_functions:
|
|
3073
|
+
if resolve:
|
|
3074
|
+
image, image_build_id = self._resolve_deployment_image(image_functions)
|
|
3075
|
+
else:
|
|
3076
|
+
image = self._unresolved_deployment_image(image_functions)
|
|
3077
|
+
|
|
3078
|
+
return DeploymentManifest(
|
|
3079
|
+
app_name=self.name,
|
|
3080
|
+
bundle_id=bundle_id,
|
|
3081
|
+
runtime={
|
|
3082
|
+
"python_version": _python_version(),
|
|
3083
|
+
"cloudpickle_version": cloudpickle_version(),
|
|
3084
|
+
},
|
|
3085
|
+
image=image,
|
|
3086
|
+
image_build_id=image_build_id,
|
|
3087
|
+
labels=labels,
|
|
3088
|
+
endpoints=endpoints,
|
|
3089
|
+
functions=functions,
|
|
3090
|
+
classes=classes,
|
|
3091
|
+
crons=crons,
|
|
3092
|
+
)
|
|
3093
|
+
|
|
3094
|
+
def _resolve_deployment_image(
|
|
3095
|
+
self, endpoint_functions: Sequence[Function]
|
|
3096
|
+
) -> tuple[Optional[str], Optional[str]]:
|
|
3097
|
+
resolved: set[tuple[str, str]] = set()
|
|
3098
|
+
for fn in endpoint_functions:
|
|
3099
|
+
image_str, image_build_id = fn._resolve_image()
|
|
3100
|
+
if image_str is not None:
|
|
3101
|
+
resolved.add(("image", image_str))
|
|
3102
|
+
elif image_build_id is not None:
|
|
3103
|
+
resolved.add(("image_build_id", image_build_id))
|
|
3104
|
+
if len(resolved) != 1:
|
|
3105
|
+
raise exc.InvalidArgumentError(
|
|
3106
|
+
"INVALID_ARGUMENT",
|
|
3107
|
+
"all deployment endpoints must use the same effective image",
|
|
3108
|
+
)
|
|
3109
|
+
kind, value = next(iter(resolved))
|
|
3110
|
+
if kind == "image":
|
|
3111
|
+
return value, None
|
|
3112
|
+
return None, value
|
|
3113
|
+
|
|
3114
|
+
def _unresolved_deployment_image(
|
|
3115
|
+
self, endpoint_functions: Sequence[Function]
|
|
3116
|
+
) -> Optional[str]:
|
|
3117
|
+
images = []
|
|
3118
|
+
for fn in endpoint_functions:
|
|
3119
|
+
img = fn.image if fn.image is not None else self.default_image
|
|
3120
|
+
if isinstance(img, str):
|
|
3121
|
+
from ._image import to_wire_image
|
|
3122
|
+
|
|
3123
|
+
images.append(to_wire_image(img))
|
|
3124
|
+
elif img is not None:
|
|
3125
|
+
return None
|
|
3126
|
+
return images[0] if images and len(set(images)) == 1 else None
|
|
3127
|
+
|
|
3128
|
+
def local_entrypoint(
|
|
3129
|
+
self, _fn: Optional[Callable[..., Any]] = None
|
|
3130
|
+
) -> Callable[..., Any]:
|
|
3131
|
+
"""Decorator: mark the function ``blade run`` should invoke locally.
|
|
3132
|
+
|
|
3133
|
+
Usable bare (``@app.local_entrypoint``) or called
|
|
3134
|
+
(``@app.local_entrypoint()``).
|
|
3135
|
+
"""
|
|
3136
|
+
|
|
3137
|
+
def wrap(fn: Callable[..., Any]) -> Callable[..., Any]:
|
|
3138
|
+
self.local_entrypoints.append(fn)
|
|
3139
|
+
return fn
|
|
3140
|
+
|
|
3141
|
+
if _fn is not None:
|
|
3142
|
+
return wrap(_fn)
|
|
3143
|
+
return wrap
|
|
3144
|
+
|
|
3145
|
+
# -- wiring used by the CLI and lifecycle ------------------------------
|
|
3146
|
+
|
|
3147
|
+
def _bind_client(self, client: Any) -> None:
|
|
3148
|
+
self._client = client
|
|
3149
|
+
|
|
3150
|
+
def _set_bundle_context(self, root: Path, entry_module: str) -> None:
|
|
3151
|
+
self._bundle_root = root
|
|
3152
|
+
self._entry_module = entry_module
|
|
3153
|
+
|
|
3154
|
+
def _module_for(self, module: str) -> str:
|
|
3155
|
+
"""Map the *defining* module to the name importable inside the bundle.
|
|
3156
|
+
|
|
3157
|
+
A function defined in the entrypoint script reports ``__main__`` when the
|
|
3158
|
+
script is executed directly; remotely it must be imported by the script's
|
|
3159
|
+
own module name (the file stem). The CLI provides that mapping.
|
|
3160
|
+
"""
|
|
3161
|
+
if module in ("__main__", "__mp_main__") and self._entry_module:
|
|
3162
|
+
return self._entry_module
|
|
3163
|
+
return module
|
|
3164
|
+
|
|
3165
|
+
def _require_client(self) -> Any:
|
|
3166
|
+
if self._client is None:
|
|
3167
|
+
from ._client import Client
|
|
3168
|
+
from .config import load_config
|
|
3169
|
+
|
|
3170
|
+
cfg = load_config()
|
|
3171
|
+
if not cfg.token:
|
|
3172
|
+
raise exc.AuthenticationError(
|
|
3173
|
+
"UNAUTHENTICATED",
|
|
3174
|
+
"no token configured: run `blade token set <token>` or set "
|
|
3175
|
+
"FABRIC_TOKEN",
|
|
3176
|
+
)
|
|
3177
|
+
self._client = Client(cfg.api_url, cfg.token)
|
|
3178
|
+
self._client.warn_if_outdated()
|
|
3179
|
+
return self._client
|
|
3180
|
+
|
|
3181
|
+
def _client_cancel(self, run_id: str) -> None:
|
|
3182
|
+
self._require_client().cancel_run(run_id)
|
|
3183
|
+
|
|
3184
|
+
def _ensure_bundle(self) -> str:
|
|
3185
|
+
"""Build (once) the project bundle and upload it, with dedup."""
|
|
3186
|
+
if self._bundle_id is not None:
|
|
3187
|
+
return self._bundle_id
|
|
3188
|
+
bundle_id, _sha = self._build_and_upload_bundle()
|
|
3189
|
+
return bundle_id
|
|
3190
|
+
|
|
3191
|
+
def _rebuild_bundle(self) -> tuple[str, str]:
|
|
3192
|
+
"""Re-bundle the project from scratch (used by ``blade serve``).
|
|
3193
|
+
|
|
3194
|
+
Returns ``(bundle_id, sha256)``. Dedup makes an unchanged project a
|
|
3195
|
+
no-op upload (same sha256 → server returns the existing bundle). Forces
|
|
3196
|
+
a fresh build (does not reuse ``self._bundle_id``) so edits are picked up.
|
|
3197
|
+
"""
|
|
3198
|
+
return self._build_and_upload_bundle(force=True)
|
|
3199
|
+
|
|
3200
|
+
def _build_and_upload_bundle(self, *, force: bool = False) -> tuple[str, str]:
|
|
3201
|
+
if not force and self._bundle_id is not None and self._bundle_sha256:
|
|
3202
|
+
return self._bundle_id, self._bundle_sha256
|
|
3203
|
+
|
|
3204
|
+
from ._bundle import build_bundle
|
|
3205
|
+
|
|
3206
|
+
root = self._bundle_root or Path.cwd()
|
|
3207
|
+
data, sha256, _arcnames = build_bundle(root)
|
|
3208
|
+
|
|
3209
|
+
# Local pre-check against the 256 MiB ceiling (conventions §6). The server
|
|
3210
|
+
# is authoritative and returns 413 BUNDLE_TOO_LARGE, but failing here avoids
|
|
3211
|
+
# paying for a large presigned upload that is bound to be rejected.
|
|
3212
|
+
from ._constants import BUNDLE_MAX_BYTES
|
|
3213
|
+
|
|
3214
|
+
if len(data) > BUNDLE_MAX_BYTES:
|
|
3215
|
+
raise exc.PayloadTooLargeError(
|
|
3216
|
+
"BUNDLE_TOO_LARGE",
|
|
3217
|
+
f"code bundle is {len(data)} bytes, over the "
|
|
3218
|
+
f"{BUNDLE_MAX_BYTES}-byte (256 MiB) limit; trim files or add them "
|
|
3219
|
+
"to .forgeignore.",
|
|
3220
|
+
)
|
|
3221
|
+
|
|
3222
|
+
client = self._require_client()
|
|
3223
|
+
resp = client.create_bundle(len(data), sha256)
|
|
3224
|
+
bundle = resp["bundle"]
|
|
3225
|
+
if not resp.get("deduplicated"):
|
|
3226
|
+
upload_url = resp.get("upload_url")
|
|
3227
|
+
if not upload_url:
|
|
3228
|
+
# Contract: a non-deduplicated bundle must carry an upload_url.
|
|
3229
|
+
# If it doesn't, the bundle stays `pending_upload` and the next
|
|
3230
|
+
# create_run fails with an opaque 409 BUNDLE_NOT_READY. Fail loud
|
|
3231
|
+
# here instead of shipping a half-created bundle id.
|
|
3232
|
+
raise exc.FabricError(
|
|
3233
|
+
f"control plane returned a non-deduplicated bundle "
|
|
3234
|
+
f"{bundle.get('bundle_id')!r} without an upload_url; cannot "
|
|
3235
|
+
"complete the bundle upload."
|
|
3236
|
+
)
|
|
3237
|
+
client.upload_to_presigned(upload_url, data, "application/gzip")
|
|
3238
|
+
bundle = client.complete_bundle(bundle["bundle_id"])
|
|
3239
|
+
self._bundle_id = bundle["bundle_id"]
|
|
3240
|
+
self._bundle_sha256 = sha256
|
|
3241
|
+
return self._bundle_id, sha256
|