seldon-sdk 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,225 @@
1
+ """Explicit module exports and content-free local/remote drift inspection."""
2
+
3
+ import importlib
4
+ import inspect
5
+ import socket
6
+ import sys
7
+ from contextlib import contextmanager
8
+ from pathlib import Path
9
+ from unittest.mock import patch
10
+
11
+ from .contract import template
12
+ from .decorator import Function
13
+ from .errors import SeldonContractError
14
+ from .files import atomic_json, digest, read_json, resolve_function
15
+ from .lockfile import locate, read, set_entry
16
+
17
+
18
+ @contextmanager
19
+ def offline_imports():
20
+ # This protects ordinary accidental network effects, not hostile Python code.
21
+ def blocked(*args, **kwargs):
22
+ raise SeldonContractError("module_network_access_forbidden")
23
+
24
+ with (
25
+ patch.object(socket.socket, "connect", blocked),
26
+ patch.object(socket.socket, "connect_ex", blocked),
27
+ patch.object(socket, "create_connection", blocked),
28
+ ):
29
+ yield
30
+
31
+
32
+ def declarations(modules):
33
+ with offline_imports():
34
+ # The explicit module belongs to the invoking application, even for a console shim.
35
+ root = str(Path.cwd())
36
+ added = root not in sys.path
37
+ if added:
38
+ sys.path.insert(0, root)
39
+ try:
40
+ return _declarations(modules)
41
+ finally:
42
+ if added:
43
+ sys.path.remove(root)
44
+
45
+
46
+ def _declarations(modules):
47
+ found = {}
48
+ for name in modules:
49
+ module = importlib.import_module(name)
50
+ for candidate in vars(module).values():
51
+ if isinstance(candidate, Function) and candidate.original.__module__ == name:
52
+ if candidate.key in found and found[candidate.key] is not candidate:
53
+ raise SeldonContractError("duplicate_function_key")
54
+ found[candidate.key] = candidate
55
+ return dict(sorted(found.items()))
56
+
57
+
58
+ def exported(modules, path):
59
+ rows, sources = [], []
60
+ for key, fn in declarations(modules).items():
61
+ if fn.example is None:
62
+ rows.append({"key": key, "error": "example_required"})
63
+ else:
64
+ sources.append(fn.export())
65
+ rows.append({"key": key, "status": "exported"})
66
+ atomic_json(path, sources)
67
+ return {"items": rows, "source_file": str(path)}, int(any("error" in row for row in rows))
68
+
69
+
70
+ def lock_set(module, key, identity):
71
+ identity = resolve_function(identity)
72
+ functions = declarations([module])
73
+ if key not in functions:
74
+ raise SeldonContractError("unknown_function_key")
75
+ fn = functions[key]
76
+ path = locate(inspect.getsourcefile(fn.original))
77
+ set_entry(
78
+ path,
79
+ key,
80
+ {
81
+ "function_id": identity,
82
+ "contract_digest": fn.digest,
83
+ "parts": contract_parts(fn.contract),
84
+ },
85
+ )
86
+ return {"key": key, "function_id": identity, "lockfile": str(path)}
87
+
88
+
89
+ def contract_parts(contract):
90
+ # Digest each field separately to name changes without persisting prompt contents.
91
+ parts = {}
92
+
93
+ def walk(value, pointer):
94
+ if isinstance(value, dict):
95
+ for key in sorted(value):
96
+ walk(value[key], pointer + "/" + key.replace("~", "~0").replace("/", "~1"))
97
+ elif isinstance(value, list):
98
+ for index, child in enumerate(value):
99
+ walk(child, pointer + "/" + str(index))
100
+ else:
101
+ parts[pointer] = digest(value)
102
+
103
+ walk(contract, "")
104
+ return parts
105
+
106
+
107
+ def check_modules(modules, *, api=None, allow_pending=False):
108
+ rows = []
109
+ for key, fn in declarations(modules).items():
110
+ path = locate(inspect.getsourcefile(fn.original))
111
+ entry = read(path)["functions"].get(key) if path.exists() else None
112
+ row = {"key": key, "status": "unregistered"}
113
+ if entry:
114
+ row["status"] = "unchanged" if entry["contract_digest"] == fn.digest else "stale_lock"
115
+ parts = contract_parts(fn.contract)
116
+ row["changed_parts"] = sorted(
117
+ pointer
118
+ for pointer in set(parts) | set(entry.get("parts", {}))
119
+ if parts.get(pointer) != entry.get("parts", {}).get(pointer)
120
+ )
121
+ if api is not None and row["status"] == "unchanged":
122
+ detail = api.detail(entry["function_id"])
123
+ active = next(
124
+ (
125
+ r
126
+ for r in detail.get("revisions", [])
127
+ if str(r["id"]) == str(detail.get("active_revision"))
128
+ ),
129
+ None,
130
+ )
131
+ if active is None:
132
+ row["status"] = "pending"
133
+ elif active.get("source_digest") != fn.digest:
134
+ row["status"] = "active_drift"
135
+ rows.append(row)
136
+ return {"items": rows}, int(
137
+ any(
138
+ row["status"] != "unchanged" and not (allow_pending and row["status"] == "pending")
139
+ for row in rows
140
+ )
141
+ )
142
+
143
+
144
+ def sync_modules(modules, api):
145
+ from .management import FUNCTIONS
146
+
147
+ # Authentication occurs before importing application code or sending any request.
148
+ api.login.access_token()
149
+ capabilities = api.request("GET", FUNCTIONS + "/sdk-capabilities")
150
+ if "seldon-sdk-contract-v1" not in capabilities.get("contract_versions", []):
151
+ raise SeldonContractError("sdk_contract_version_unsupported")
152
+ functions = declarations(modules)
153
+ sources = [fn.export() for fn in functions.values()]
154
+ if not 1 <= len(sources) <= 50:
155
+ raise SeldonContractError("sync_batch_too_large")
156
+ result = api.request("POST", FUNCTIONS + "/sdk-sync", {"contracts": sources})
157
+ for row in result["items"]:
158
+ if not row.get("error") and row["key"] in functions:
159
+ fn = functions[row["key"]]
160
+ set_entry(
161
+ locate(inspect.getsourcefile(fn.original)),
162
+ fn.key,
163
+ {
164
+ "function_id": resolve_function(row["function_id"]),
165
+ "contract_digest": fn.digest,
166
+ "parts": contract_parts(fn.contract),
167
+ },
168
+ )
169
+ return result, int(any(row.get("error") for row in result["items"]))
170
+
171
+
172
+ def check_template(api, function, path, syntax):
173
+ import re
174
+ import string
175
+
176
+ identity = resolve_function(function)
177
+ remote = api.request("GET", "/v1/seldon/functions/" + identity + "/contract")
178
+ data = read_json(path) if Path(path).suffix == ".json" else Path(path).read_text()
179
+ messages = (
180
+ data.get("messages") if isinstance(data, dict) else [{"role": "user", "content": data}]
181
+ )
182
+ changed = []
183
+ if remote.get("template_syntax") != syntax:
184
+ changed.append("syntax")
185
+ for index, message in enumerate(messages):
186
+ content = message["content"]
187
+ if syntax == "python_format":
188
+ parsed = list(string.Formatter().parse(content))
189
+ variables = [field for _, field, _, _ in parsed if field is not None]
190
+ template(content, variables)
191
+ literals = [""]
192
+ for literal, field, _, _ in parsed:
193
+ literals[-1] += literal
194
+ if field is not None:
195
+ literals.append("")
196
+ else:
197
+ tokens = list(re.finditer(r"{{\s*([A-Za-z]\w*)\s*}}", content))
198
+ variables = [token[1] for token in tokens]
199
+ literals, offset = [], 0
200
+ for token in tokens:
201
+ literals.append(content[offset : token.start()])
202
+ offset = token.end()
203
+ literals.append(content[offset:])
204
+ if any(mark in "".join(literals) for mark in ("{{", "}}", "{%", "{#")):
205
+ raise SeldonContractError("template_syntax_unsupported")
206
+ expected = remote["messages"][index] if index < len(remote["messages"]) else {}
207
+ if expected.get("role") != message["role"]:
208
+ changed.append(f"messages/{index}/role")
209
+ if expected.get("variables") != variables:
210
+ changed.append(f"messages/{index}/variables")
211
+ segments = [digest(literal) for literal in literals]
212
+ for segment in range(max(len(segments), len(expected.get("segment_digests", [])))):
213
+ if (
214
+ segment >= len(segments)
215
+ or segment >= len(expected.get("segment_digests", []))
216
+ or segments[segment] != expected["segment_digests"][segment]
217
+ ):
218
+ changed.append(f"messages/{index}/segments/{segment}")
219
+ if len(messages) != len(remote["messages"]):
220
+ changed.append("messages/count")
221
+ return {
222
+ "function_id": identity,
223
+ "status": "changed" if changed else "unchanged",
224
+ "changed_parts": changed,
225
+ }, int(bool(changed))
seldon_sdk/py.typed ADDED
File without changes
seldon_sdk/runtime.py ADDED
@@ -0,0 +1,375 @@
1
+ """Typed serving with bounded retries and explicit reference fallback."""
2
+
3
+ import asyncio
4
+ import inspect
5
+ import logging
6
+ import math
7
+ import os
8
+ import platform
9
+ import threading
10
+ import time
11
+ import warnings
12
+ import weakref
13
+ from collections import deque
14
+ from dataclasses import dataclass
15
+ from urllib.parse import urlsplit
16
+ from uuid import uuid4
17
+
18
+ import httpx
19
+ from pydantic import ValidationError
20
+
21
+ from . import __version__
22
+ from .contract import render
23
+ from .errors import (
24
+ SeldonAuthError,
25
+ SeldonConfigError,
26
+ SeldonContractError,
27
+ SeldonError,
28
+ SeldonOutputValidationError,
29
+ SeldonReferenceError,
30
+ SeldonSpendError,
31
+ )
32
+ from .files import canonical, safe_code
33
+ from .lockfile import locate, read
34
+
35
+ _settings = {}
36
+ _async_clients = weakref.WeakKeyDictionary()
37
+ _sync = threading.local()
38
+ _guard = threading.Lock()
39
+ _warned = set()
40
+
41
+
42
+ def configure(**settings):
43
+ allowed = {
44
+ "base_url",
45
+ "api_key",
46
+ "reference_base_url",
47
+ "reference_api_key",
48
+ "timeout",
49
+ "reference_timeout",
50
+ "fallback",
51
+ "on_drift",
52
+ "on_invalid_output",
53
+ "on_call",
54
+ "breaker",
55
+ "http_client",
56
+ "async_http_client",
57
+ }
58
+ if set(settings) - allowed:
59
+ raise SeldonConfigError("unknown_configuration")
60
+ _settings.update(settings)
61
+
62
+
63
+ def _url(value):
64
+ parsed = urlsplit(value)
65
+ if (
66
+ (
67
+ parsed.scheme != "https"
68
+ and not (parsed.scheme == "http" and parsed.hostname == "localhost")
69
+ )
70
+ or not parsed.hostname
71
+ or parsed.username
72
+ or parsed.password
73
+ or parsed.query
74
+ or parsed.fragment
75
+ ):
76
+ raise SeldonConfigError("https_endpoint_required")
77
+ value = value.rstrip("/")
78
+ return value if value.endswith("/v1") else value + "/v1"
79
+
80
+
81
+ def warn_once(key, code):
82
+ with _guard:
83
+ if (key, code) in _warned:
84
+ return
85
+ _warned.add((key, code))
86
+ warnings.warn(code, RuntimeWarning, stacklevel=3)
87
+
88
+
89
+ @dataclass
90
+ class CallInfo:
91
+ path: str = "function"
92
+ reason: str | None = None
93
+ function_id: str | None = None
94
+ revision_id: str | None = None
95
+ execution_id: str | None = None
96
+ server_fallback_used: bool = False
97
+ contract_check: str = "unsupported"
98
+ attempts: int = 0
99
+ latency_ms: float = 0
100
+ usage: dict | None = None
101
+ reference_output_fenced: bool = False
102
+
103
+
104
+ class Breaker:
105
+ def __init__(self, clock=time.monotonic):
106
+ self.clock = clock
107
+ self.events = deque(maxlen=20)
108
+ self.until = 0
109
+ self.probing = False
110
+ self.lock = threading.Lock()
111
+
112
+ def allow(self):
113
+ with self.lock:
114
+ if not self.until:
115
+ return "normal"
116
+ if self.clock() < self.until or self.probing:
117
+ return False
118
+ self.probing = True
119
+ return "probe"
120
+
121
+ def record(self, failed, *, probe=False):
122
+ with self.lock:
123
+ now = self.clock()
124
+ if probe:
125
+ self.probing = False
126
+ self.until = now + 60 if failed else 0
127
+ self.events.clear()
128
+ return
129
+ self.events.append((now, failed))
130
+ if sum(failure for at, failure in self.events if now - at <= 300) >= 10:
131
+ self.until = now + 60
132
+
133
+ def release_probe(self):
134
+ with self.lock:
135
+ self.probing = False
136
+
137
+
138
+ def _client(async_mode, options):
139
+ supplied = options.get("async_http_client" if async_mode else "http_client")
140
+ if supplied is not None:
141
+ return supplied
142
+ if not async_mode:
143
+ if not hasattr(_sync, "client"):
144
+ _sync.client = httpx.Client(follow_redirects=False)
145
+ return _sync.client
146
+ loop = asyncio.get_running_loop()
147
+ with _guard:
148
+ if loop not in _async_clients:
149
+ _async_clients[loop] = httpx.AsyncClient(follow_redirects=False)
150
+ return _async_clients[loop]
151
+
152
+
153
+ async def aclose():
154
+ with _guard:
155
+ client = _async_clients.pop(asyncio.get_running_loop(), None)
156
+ if client is not None:
157
+ await client.aclose()
158
+
159
+
160
+ def close():
161
+ if hasattr(_sync, "client"):
162
+ _sync.client.close()
163
+ del _sync.client
164
+
165
+
166
+ def options_for(fn):
167
+ values = {
168
+ "base_url": os.environ.get("SELDON_BASE_URL", "https://api.seldon-ai.com"),
169
+ "api_key": os.environ.get("SELDON_API_KEY"),
170
+ "timeout": 58,
171
+ "reference_timeout": 120,
172
+ "fallback": "reference",
173
+ "on_drift": "reference",
174
+ "on_invalid_output": "raise",
175
+ "breaker": True,
176
+ }
177
+ values.update(_settings)
178
+ values.update(fn.options)
179
+ for field, allowed in (
180
+ ("fallback", {"reference", "never"}),
181
+ ("on_drift", {"reference", "raise"}),
182
+ ("on_invalid_output", {"reference", "raise"}),
183
+ ):
184
+ if values[field] not in allowed:
185
+ raise SeldonConfigError("invalid_" + field)
186
+ values["base_url"] = _url(values["base_url"])
187
+ if not values.get("api_key"):
188
+ raise SeldonConfigError("api_key_required")
189
+ if values["timeout"] <= 55:
190
+ warn_once(fn.key, "timeout_below_function_deadline")
191
+ return values
192
+
193
+
194
+ def resolve_identity(fn):
195
+ if fn.id:
196
+ return fn.id, None
197
+ path = locate(inspect.getsourcefile(fn.original))
198
+ if not path.exists():
199
+ return None, "not_registered"
200
+ entry = read(path)["functions"].get(fn.key)
201
+ if not entry:
202
+ return None, "not_registered"
203
+ if entry["contract_digest"] != fn.digest:
204
+ return None, "lockfile_stale"
205
+ return entry["function_id"], None
206
+
207
+
208
+ def _error(response):
209
+ try:
210
+ body = response.json()
211
+ code = body.get("error", "service_error")
212
+ if isinstance(code, dict):
213
+ code = code.get("code")
214
+ return safe_code(code)
215
+ except (ValueError, AttributeError):
216
+ return "service_error"
217
+
218
+
219
+ def _raise_response(response):
220
+ code, status = _error(response), response.status_code
221
+ kind = (
222
+ SeldonAuthError
223
+ if status in (401, 403)
224
+ else SeldonSpendError
225
+ if status == 402
226
+ else SeldonContractError
227
+ if status in (400, 422)
228
+ else SeldonError
229
+ )
230
+ raise kind(code, status=status)
231
+
232
+
233
+ async def invoke(fn, inputs, *, async_mode):
234
+ started = time.monotonic()
235
+ info = CallInfo()
236
+ options = options_for(fn)
237
+ admitted = False
238
+ client = _client(async_mode, options)
239
+
240
+ async def request(url, body, key, timeout, headers=None):
241
+ kwargs = {
242
+ "json": body,
243
+ "headers": {
244
+ "Authorization": "Bearer " + key,
245
+ "User-Agent": f"seldon-sdk/{__version__} python/{platform.python_version()}",
246
+ **(headers or {}),
247
+ },
248
+ "timeout": timeout,
249
+ "follow_redirects": False,
250
+ }
251
+ result = client.post(url + "/chat/completions", **kwargs)
252
+ return await result if inspect.isawaitable(result) else result
253
+
254
+ async def reference(reason, cause=None):
255
+ info.path, info.reason = "reference", reason
256
+ if options["fallback"] == "never":
257
+ raise cause or SeldonReferenceError(reason)
258
+ origin = _url(options.get("reference_base_url", options["base_url"]))
259
+ different = urlsplit(origin).netloc != urlsplit(options["base_url"]).netloc
260
+ key = options.get("reference_api_key") or (None if different else options["api_key"])
261
+ if not key:
262
+ raise SeldonConfigError("reference_api_key_required")
263
+ body = {
264
+ key: fn.contract[key]
265
+ for key in ("model", "temperature", "max_tokens", "response_format")
266
+ }
267
+ body["messages"] = render(fn.contract, inputs)
268
+ try:
269
+ response = await request(origin, body, key, options["reference_timeout"])
270
+ if response.status_code != 200:
271
+ _raise_response(response)
272
+ data = response.json()
273
+ content = data["choices"][0]["message"]["content"]
274
+ if content.strip().startswith("```json") and content.strip().endswith("```"):
275
+ content = content.strip()[7:-3].strip()
276
+ info.reference_output_fenced = True
277
+ info.usage = data.get("usage")
278
+ return fn.output.validate_json(content)
279
+ except (httpx.HTTPError, ValueError, TypeError, KeyError, IndexError, ValidationError):
280
+ raise SeldonReferenceError("reference_output_invalid") from cause
281
+
282
+ try:
283
+ identity, reason = resolve_identity(fn)
284
+ info.function_id = identity
285
+ if os.environ.get("SELDON_MODE") == "reference":
286
+ return await reference("reference_mode"), info
287
+ if reason:
288
+ warn_once(fn.key, reason + ": run seldon sync")
289
+ return await reference(reason), info
290
+ if options["breaker"]:
291
+ admitted = fn.breaker.allow()
292
+ if not admitted:
293
+ return await reference("circuit_open"), info
294
+ headers = {"Idempotency-Key": str(uuid4()), "X-Seldon-Contract-Digest": fn.digest}
295
+ body = {
296
+ "model": "seldon/function/" + identity,
297
+ "messages": [{"role": "user", "content": canonical(inputs)}],
298
+ }
299
+ for attempt in range(3):
300
+ info.attempts = attempt + 1
301
+ try:
302
+ response = await request(
303
+ options["base_url"], body, options["api_key"], options["timeout"], headers
304
+ )
305
+ except (httpx.ConnectError, httpx.ReadTimeout):
306
+ if attempt < 2:
307
+ await asyncio.sleep(0.1 * (2**attempt))
308
+ continue
309
+ return await reference(
310
+ "transport_unavailable", SeldonError("transport_unavailable")
311
+ ), info
312
+ status, code = response.status_code, _error(response)
313
+ if status == 200:
314
+ fn.breaker.record(False, probe=admitted == "probe")
315
+ try:
316
+ data = response.json()
317
+ except ValueError:
318
+ raise SeldonOutputValidationError("invalid_function_output") from None
319
+ meta = data.get("seldon", {})
320
+ info.revision_id, info.execution_id = (
321
+ meta.get("revision_id"),
322
+ meta.get("execution_id"),
323
+ )
324
+ info.server_fallback_used = bool(meta.get("fallback_used"))
325
+ info.contract_check = response.headers.get("X-Seldon-Contract-Check", "unsupported")
326
+ info.usage = data.get("usage")
327
+ try:
328
+ result = fn.output.validate_json(data["choices"][0]["message"]["content"])
329
+ except (ValueError, ValidationError, KeyError, IndexError, TypeError):
330
+ if options["on_invalid_output"] == "reference":
331
+ return await reference("invalid_function_output"), info
332
+ raise SeldonOutputValidationError("invalid_function_output") from None
333
+ return result, info
334
+ if status in (401, 402, 403) or status == 429 and code == "api_key_rate_limit":
335
+ _raise_response(response)
336
+ if status == 400 or status == 422:
337
+ _raise_response(response)
338
+ if status == 409 and code == "contract_digest_mismatch":
339
+ warn_once(fn.key, "contract_digest_mismatch: run seldon sync")
340
+ if options["on_drift"] == "raise":
341
+ raise SeldonContractError(code)
342
+ return await reference(code), info
343
+ if status in (502, 504):
344
+ fn.breaker.record(True, probe=admitted == "probe")
345
+ return await reference(
346
+ "server_execution_failed" if status == 502 else "server_deadline_exceeded"
347
+ ), info
348
+ if (status == 429 and code == "execution_capacity") or (
349
+ status == 409 and code == "execution_in_progress"
350
+ ):
351
+ if attempt == 2:
352
+ return await reference(code), info
353
+ try:
354
+ pause = float(response.headers.get("Retry-After", "0.1"))
355
+ pause = min(5, max(0, pause)) if math.isfinite(pause) else 0.1
356
+ except ValueError:
357
+ pause = 0.1
358
+ await asyncio.sleep(pause)
359
+ continue
360
+ if (status == 404 and code in {"function_not_found", "revision_not_available"}) or (
361
+ status == 409 and code in {"function_disabled", "execution_disabled"}
362
+ ):
363
+ return await reference(code), info
364
+ _raise_response(response)
365
+ raise SeldonError("execution_unavailable")
366
+ finally:
367
+ if admitted == "probe":
368
+ fn.breaker.release_probe()
369
+ info.latency_ms = (time.monotonic() - started) * 1000
370
+ hook = options.get("on_call")
371
+ if hook:
372
+ try:
373
+ hook(info)
374
+ except Exception:
375
+ logging.getLogger(__name__).warning("call_metadata_hook_failed")