seldon-sdk 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seldon_sdk/__init__.py +23 -0
- seldon_sdk/__init__.pyi +30 -0
- seldon_sdk/auth.py +286 -0
- seldon_sdk/cli.py +344 -0
- seldon_sdk/contract.py +200 -0
- seldon_sdk/decorator.py +114 -0
- seldon_sdk/errors.py +37 -0
- seldon_sdk/files.py +350 -0
- seldon_sdk/local.py +69 -0
- seldon_sdk/lockfile.py +42 -0
- seldon_sdk/management.py +532 -0
- seldon_sdk/preparation.py +225 -0
- seldon_sdk/py.typed +0 -0
- seldon_sdk/runtime.py +375 -0
- seldon_sdk/scanner.py +143 -0
- seldon_sdk/skill/SKILL.md +61 -0
- seldon_sdk/testing.py +32 -0
- seldon_sdk/testing.pyi +12 -0
- seldon_sdk-0.1.0.dev0.dist-info/METADATA +263 -0
- seldon_sdk-0.1.0.dev0.dist-info/RECORD +23 -0
- seldon_sdk-0.1.0.dev0.dist-info/WHEEL +4 -0
- seldon_sdk-0.1.0.dev0.dist-info/entry_points.txt +2 -0
- seldon_sdk-0.1.0.dev0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""Explicit module exports and content-free local/remote drift inspection."""
|
|
2
|
+
|
|
3
|
+
import importlib
|
|
4
|
+
import inspect
|
|
5
|
+
import socket
|
|
6
|
+
import sys
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from unittest.mock import patch
|
|
10
|
+
|
|
11
|
+
from .contract import template
|
|
12
|
+
from .decorator import Function
|
|
13
|
+
from .errors import SeldonContractError
|
|
14
|
+
from .files import atomic_json, digest, read_json, resolve_function
|
|
15
|
+
from .lockfile import locate, read, set_entry
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@contextmanager
|
|
19
|
+
def offline_imports():
|
|
20
|
+
# This protects ordinary accidental network effects, not hostile Python code.
|
|
21
|
+
def blocked(*args, **kwargs):
|
|
22
|
+
raise SeldonContractError("module_network_access_forbidden")
|
|
23
|
+
|
|
24
|
+
with (
|
|
25
|
+
patch.object(socket.socket, "connect", blocked),
|
|
26
|
+
patch.object(socket.socket, "connect_ex", blocked),
|
|
27
|
+
patch.object(socket, "create_connection", blocked),
|
|
28
|
+
):
|
|
29
|
+
yield
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def declarations(modules):
|
|
33
|
+
with offline_imports():
|
|
34
|
+
# The explicit module belongs to the invoking application, even for a console shim.
|
|
35
|
+
root = str(Path.cwd())
|
|
36
|
+
added = root not in sys.path
|
|
37
|
+
if added:
|
|
38
|
+
sys.path.insert(0, root)
|
|
39
|
+
try:
|
|
40
|
+
return _declarations(modules)
|
|
41
|
+
finally:
|
|
42
|
+
if added:
|
|
43
|
+
sys.path.remove(root)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _declarations(modules):
|
|
47
|
+
found = {}
|
|
48
|
+
for name in modules:
|
|
49
|
+
module = importlib.import_module(name)
|
|
50
|
+
for candidate in vars(module).values():
|
|
51
|
+
if isinstance(candidate, Function) and candidate.original.__module__ == name:
|
|
52
|
+
if candidate.key in found and found[candidate.key] is not candidate:
|
|
53
|
+
raise SeldonContractError("duplicate_function_key")
|
|
54
|
+
found[candidate.key] = candidate
|
|
55
|
+
return dict(sorted(found.items()))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def exported(modules, path):
|
|
59
|
+
rows, sources = [], []
|
|
60
|
+
for key, fn in declarations(modules).items():
|
|
61
|
+
if fn.example is None:
|
|
62
|
+
rows.append({"key": key, "error": "example_required"})
|
|
63
|
+
else:
|
|
64
|
+
sources.append(fn.export())
|
|
65
|
+
rows.append({"key": key, "status": "exported"})
|
|
66
|
+
atomic_json(path, sources)
|
|
67
|
+
return {"items": rows, "source_file": str(path)}, int(any("error" in row for row in rows))
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def lock_set(module, key, identity):
|
|
71
|
+
identity = resolve_function(identity)
|
|
72
|
+
functions = declarations([module])
|
|
73
|
+
if key not in functions:
|
|
74
|
+
raise SeldonContractError("unknown_function_key")
|
|
75
|
+
fn = functions[key]
|
|
76
|
+
path = locate(inspect.getsourcefile(fn.original))
|
|
77
|
+
set_entry(
|
|
78
|
+
path,
|
|
79
|
+
key,
|
|
80
|
+
{
|
|
81
|
+
"function_id": identity,
|
|
82
|
+
"contract_digest": fn.digest,
|
|
83
|
+
"parts": contract_parts(fn.contract),
|
|
84
|
+
},
|
|
85
|
+
)
|
|
86
|
+
return {"key": key, "function_id": identity, "lockfile": str(path)}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def contract_parts(contract):
|
|
90
|
+
# Digest each field separately to name changes without persisting prompt contents.
|
|
91
|
+
parts = {}
|
|
92
|
+
|
|
93
|
+
def walk(value, pointer):
|
|
94
|
+
if isinstance(value, dict):
|
|
95
|
+
for key in sorted(value):
|
|
96
|
+
walk(value[key], pointer + "/" + key.replace("~", "~0").replace("/", "~1"))
|
|
97
|
+
elif isinstance(value, list):
|
|
98
|
+
for index, child in enumerate(value):
|
|
99
|
+
walk(child, pointer + "/" + str(index))
|
|
100
|
+
else:
|
|
101
|
+
parts[pointer] = digest(value)
|
|
102
|
+
|
|
103
|
+
walk(contract, "")
|
|
104
|
+
return parts
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def check_modules(modules, *, api=None, allow_pending=False):
|
|
108
|
+
rows = []
|
|
109
|
+
for key, fn in declarations(modules).items():
|
|
110
|
+
path = locate(inspect.getsourcefile(fn.original))
|
|
111
|
+
entry = read(path)["functions"].get(key) if path.exists() else None
|
|
112
|
+
row = {"key": key, "status": "unregistered"}
|
|
113
|
+
if entry:
|
|
114
|
+
row["status"] = "unchanged" if entry["contract_digest"] == fn.digest else "stale_lock"
|
|
115
|
+
parts = contract_parts(fn.contract)
|
|
116
|
+
row["changed_parts"] = sorted(
|
|
117
|
+
pointer
|
|
118
|
+
for pointer in set(parts) | set(entry.get("parts", {}))
|
|
119
|
+
if parts.get(pointer) != entry.get("parts", {}).get(pointer)
|
|
120
|
+
)
|
|
121
|
+
if api is not None and row["status"] == "unchanged":
|
|
122
|
+
detail = api.detail(entry["function_id"])
|
|
123
|
+
active = next(
|
|
124
|
+
(
|
|
125
|
+
r
|
|
126
|
+
for r in detail.get("revisions", [])
|
|
127
|
+
if str(r["id"]) == str(detail.get("active_revision"))
|
|
128
|
+
),
|
|
129
|
+
None,
|
|
130
|
+
)
|
|
131
|
+
if active is None:
|
|
132
|
+
row["status"] = "pending"
|
|
133
|
+
elif active.get("source_digest") != fn.digest:
|
|
134
|
+
row["status"] = "active_drift"
|
|
135
|
+
rows.append(row)
|
|
136
|
+
return {"items": rows}, int(
|
|
137
|
+
any(
|
|
138
|
+
row["status"] != "unchanged" and not (allow_pending and row["status"] == "pending")
|
|
139
|
+
for row in rows
|
|
140
|
+
)
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def sync_modules(modules, api):
|
|
145
|
+
from .management import FUNCTIONS
|
|
146
|
+
|
|
147
|
+
# Authentication occurs before importing application code or sending any request.
|
|
148
|
+
api.login.access_token()
|
|
149
|
+
capabilities = api.request("GET", FUNCTIONS + "/sdk-capabilities")
|
|
150
|
+
if "seldon-sdk-contract-v1" not in capabilities.get("contract_versions", []):
|
|
151
|
+
raise SeldonContractError("sdk_contract_version_unsupported")
|
|
152
|
+
functions = declarations(modules)
|
|
153
|
+
sources = [fn.export() for fn in functions.values()]
|
|
154
|
+
if not 1 <= len(sources) <= 50:
|
|
155
|
+
raise SeldonContractError("sync_batch_too_large")
|
|
156
|
+
result = api.request("POST", FUNCTIONS + "/sdk-sync", {"contracts": sources})
|
|
157
|
+
for row in result["items"]:
|
|
158
|
+
if not row.get("error") and row["key"] in functions:
|
|
159
|
+
fn = functions[row["key"]]
|
|
160
|
+
set_entry(
|
|
161
|
+
locate(inspect.getsourcefile(fn.original)),
|
|
162
|
+
fn.key,
|
|
163
|
+
{
|
|
164
|
+
"function_id": resolve_function(row["function_id"]),
|
|
165
|
+
"contract_digest": fn.digest,
|
|
166
|
+
"parts": contract_parts(fn.contract),
|
|
167
|
+
},
|
|
168
|
+
)
|
|
169
|
+
return result, int(any(row.get("error") for row in result["items"]))
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def check_template(api, function, path, syntax):
|
|
173
|
+
import re
|
|
174
|
+
import string
|
|
175
|
+
|
|
176
|
+
identity = resolve_function(function)
|
|
177
|
+
remote = api.request("GET", "/v1/seldon/functions/" + identity + "/contract")
|
|
178
|
+
data = read_json(path) if Path(path).suffix == ".json" else Path(path).read_text()
|
|
179
|
+
messages = (
|
|
180
|
+
data.get("messages") if isinstance(data, dict) else [{"role": "user", "content": data}]
|
|
181
|
+
)
|
|
182
|
+
changed = []
|
|
183
|
+
if remote.get("template_syntax") != syntax:
|
|
184
|
+
changed.append("syntax")
|
|
185
|
+
for index, message in enumerate(messages):
|
|
186
|
+
content = message["content"]
|
|
187
|
+
if syntax == "python_format":
|
|
188
|
+
parsed = list(string.Formatter().parse(content))
|
|
189
|
+
variables = [field for _, field, _, _ in parsed if field is not None]
|
|
190
|
+
template(content, variables)
|
|
191
|
+
literals = [""]
|
|
192
|
+
for literal, field, _, _ in parsed:
|
|
193
|
+
literals[-1] += literal
|
|
194
|
+
if field is not None:
|
|
195
|
+
literals.append("")
|
|
196
|
+
else:
|
|
197
|
+
tokens = list(re.finditer(r"{{\s*([A-Za-z]\w*)\s*}}", content))
|
|
198
|
+
variables = [token[1] for token in tokens]
|
|
199
|
+
literals, offset = [], 0
|
|
200
|
+
for token in tokens:
|
|
201
|
+
literals.append(content[offset : token.start()])
|
|
202
|
+
offset = token.end()
|
|
203
|
+
literals.append(content[offset:])
|
|
204
|
+
if any(mark in "".join(literals) for mark in ("{{", "}}", "{%", "{#")):
|
|
205
|
+
raise SeldonContractError("template_syntax_unsupported")
|
|
206
|
+
expected = remote["messages"][index] if index < len(remote["messages"]) else {}
|
|
207
|
+
if expected.get("role") != message["role"]:
|
|
208
|
+
changed.append(f"messages/{index}/role")
|
|
209
|
+
if expected.get("variables") != variables:
|
|
210
|
+
changed.append(f"messages/{index}/variables")
|
|
211
|
+
segments = [digest(literal) for literal in literals]
|
|
212
|
+
for segment in range(max(len(segments), len(expected.get("segment_digests", [])))):
|
|
213
|
+
if (
|
|
214
|
+
segment >= len(segments)
|
|
215
|
+
or segment >= len(expected.get("segment_digests", []))
|
|
216
|
+
or segments[segment] != expected["segment_digests"][segment]
|
|
217
|
+
):
|
|
218
|
+
changed.append(f"messages/{index}/segments/{segment}")
|
|
219
|
+
if len(messages) != len(remote["messages"]):
|
|
220
|
+
changed.append("messages/count")
|
|
221
|
+
return {
|
|
222
|
+
"function_id": identity,
|
|
223
|
+
"status": "changed" if changed else "unchanged",
|
|
224
|
+
"changed_parts": changed,
|
|
225
|
+
}, int(bool(changed))
|
seldon_sdk/py.typed
ADDED
|
File without changes
|
seldon_sdk/runtime.py
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
"""Typed serving with bounded retries and explicit reference fallback."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import inspect
|
|
5
|
+
import logging
|
|
6
|
+
import math
|
|
7
|
+
import os
|
|
8
|
+
import platform
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
import warnings
|
|
12
|
+
import weakref
|
|
13
|
+
from collections import deque
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from urllib.parse import urlsplit
|
|
16
|
+
from uuid import uuid4
|
|
17
|
+
|
|
18
|
+
import httpx
|
|
19
|
+
from pydantic import ValidationError
|
|
20
|
+
|
|
21
|
+
from . import __version__
|
|
22
|
+
from .contract import render
|
|
23
|
+
from .errors import (
|
|
24
|
+
SeldonAuthError,
|
|
25
|
+
SeldonConfigError,
|
|
26
|
+
SeldonContractError,
|
|
27
|
+
SeldonError,
|
|
28
|
+
SeldonOutputValidationError,
|
|
29
|
+
SeldonReferenceError,
|
|
30
|
+
SeldonSpendError,
|
|
31
|
+
)
|
|
32
|
+
from .files import canonical, safe_code
|
|
33
|
+
from .lockfile import locate, read
|
|
34
|
+
|
|
35
|
+
_settings = {}
|
|
36
|
+
_async_clients = weakref.WeakKeyDictionary()
|
|
37
|
+
_sync = threading.local()
|
|
38
|
+
_guard = threading.Lock()
|
|
39
|
+
_warned = set()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def configure(**settings):
|
|
43
|
+
allowed = {
|
|
44
|
+
"base_url",
|
|
45
|
+
"api_key",
|
|
46
|
+
"reference_base_url",
|
|
47
|
+
"reference_api_key",
|
|
48
|
+
"timeout",
|
|
49
|
+
"reference_timeout",
|
|
50
|
+
"fallback",
|
|
51
|
+
"on_drift",
|
|
52
|
+
"on_invalid_output",
|
|
53
|
+
"on_call",
|
|
54
|
+
"breaker",
|
|
55
|
+
"http_client",
|
|
56
|
+
"async_http_client",
|
|
57
|
+
}
|
|
58
|
+
if set(settings) - allowed:
|
|
59
|
+
raise SeldonConfigError("unknown_configuration")
|
|
60
|
+
_settings.update(settings)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _url(value):
|
|
64
|
+
parsed = urlsplit(value)
|
|
65
|
+
if (
|
|
66
|
+
(
|
|
67
|
+
parsed.scheme != "https"
|
|
68
|
+
and not (parsed.scheme == "http" and parsed.hostname == "localhost")
|
|
69
|
+
)
|
|
70
|
+
or not parsed.hostname
|
|
71
|
+
or parsed.username
|
|
72
|
+
or parsed.password
|
|
73
|
+
or parsed.query
|
|
74
|
+
or parsed.fragment
|
|
75
|
+
):
|
|
76
|
+
raise SeldonConfigError("https_endpoint_required")
|
|
77
|
+
value = value.rstrip("/")
|
|
78
|
+
return value if value.endswith("/v1") else value + "/v1"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def warn_once(key, code):
|
|
82
|
+
with _guard:
|
|
83
|
+
if (key, code) in _warned:
|
|
84
|
+
return
|
|
85
|
+
_warned.add((key, code))
|
|
86
|
+
warnings.warn(code, RuntimeWarning, stacklevel=3)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass
|
|
90
|
+
class CallInfo:
|
|
91
|
+
path: str = "function"
|
|
92
|
+
reason: str | None = None
|
|
93
|
+
function_id: str | None = None
|
|
94
|
+
revision_id: str | None = None
|
|
95
|
+
execution_id: str | None = None
|
|
96
|
+
server_fallback_used: bool = False
|
|
97
|
+
contract_check: str = "unsupported"
|
|
98
|
+
attempts: int = 0
|
|
99
|
+
latency_ms: float = 0
|
|
100
|
+
usage: dict | None = None
|
|
101
|
+
reference_output_fenced: bool = False
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class Breaker:
|
|
105
|
+
def __init__(self, clock=time.monotonic):
|
|
106
|
+
self.clock = clock
|
|
107
|
+
self.events = deque(maxlen=20)
|
|
108
|
+
self.until = 0
|
|
109
|
+
self.probing = False
|
|
110
|
+
self.lock = threading.Lock()
|
|
111
|
+
|
|
112
|
+
def allow(self):
|
|
113
|
+
with self.lock:
|
|
114
|
+
if not self.until:
|
|
115
|
+
return "normal"
|
|
116
|
+
if self.clock() < self.until or self.probing:
|
|
117
|
+
return False
|
|
118
|
+
self.probing = True
|
|
119
|
+
return "probe"
|
|
120
|
+
|
|
121
|
+
def record(self, failed, *, probe=False):
|
|
122
|
+
with self.lock:
|
|
123
|
+
now = self.clock()
|
|
124
|
+
if probe:
|
|
125
|
+
self.probing = False
|
|
126
|
+
self.until = now + 60 if failed else 0
|
|
127
|
+
self.events.clear()
|
|
128
|
+
return
|
|
129
|
+
self.events.append((now, failed))
|
|
130
|
+
if sum(failure for at, failure in self.events if now - at <= 300) >= 10:
|
|
131
|
+
self.until = now + 60
|
|
132
|
+
|
|
133
|
+
def release_probe(self):
|
|
134
|
+
with self.lock:
|
|
135
|
+
self.probing = False
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _client(async_mode, options):
|
|
139
|
+
supplied = options.get("async_http_client" if async_mode else "http_client")
|
|
140
|
+
if supplied is not None:
|
|
141
|
+
return supplied
|
|
142
|
+
if not async_mode:
|
|
143
|
+
if not hasattr(_sync, "client"):
|
|
144
|
+
_sync.client = httpx.Client(follow_redirects=False)
|
|
145
|
+
return _sync.client
|
|
146
|
+
loop = asyncio.get_running_loop()
|
|
147
|
+
with _guard:
|
|
148
|
+
if loop not in _async_clients:
|
|
149
|
+
_async_clients[loop] = httpx.AsyncClient(follow_redirects=False)
|
|
150
|
+
return _async_clients[loop]
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
async def aclose():
|
|
154
|
+
with _guard:
|
|
155
|
+
client = _async_clients.pop(asyncio.get_running_loop(), None)
|
|
156
|
+
if client is not None:
|
|
157
|
+
await client.aclose()
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def close():
|
|
161
|
+
if hasattr(_sync, "client"):
|
|
162
|
+
_sync.client.close()
|
|
163
|
+
del _sync.client
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def options_for(fn):
|
|
167
|
+
values = {
|
|
168
|
+
"base_url": os.environ.get("SELDON_BASE_URL", "https://api.seldon-ai.com"),
|
|
169
|
+
"api_key": os.environ.get("SELDON_API_KEY"),
|
|
170
|
+
"timeout": 58,
|
|
171
|
+
"reference_timeout": 120,
|
|
172
|
+
"fallback": "reference",
|
|
173
|
+
"on_drift": "reference",
|
|
174
|
+
"on_invalid_output": "raise",
|
|
175
|
+
"breaker": True,
|
|
176
|
+
}
|
|
177
|
+
values.update(_settings)
|
|
178
|
+
values.update(fn.options)
|
|
179
|
+
for field, allowed in (
|
|
180
|
+
("fallback", {"reference", "never"}),
|
|
181
|
+
("on_drift", {"reference", "raise"}),
|
|
182
|
+
("on_invalid_output", {"reference", "raise"}),
|
|
183
|
+
):
|
|
184
|
+
if values[field] not in allowed:
|
|
185
|
+
raise SeldonConfigError("invalid_" + field)
|
|
186
|
+
values["base_url"] = _url(values["base_url"])
|
|
187
|
+
if not values.get("api_key"):
|
|
188
|
+
raise SeldonConfigError("api_key_required")
|
|
189
|
+
if values["timeout"] <= 55:
|
|
190
|
+
warn_once(fn.key, "timeout_below_function_deadline")
|
|
191
|
+
return values
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def resolve_identity(fn):
|
|
195
|
+
if fn.id:
|
|
196
|
+
return fn.id, None
|
|
197
|
+
path = locate(inspect.getsourcefile(fn.original))
|
|
198
|
+
if not path.exists():
|
|
199
|
+
return None, "not_registered"
|
|
200
|
+
entry = read(path)["functions"].get(fn.key)
|
|
201
|
+
if not entry:
|
|
202
|
+
return None, "not_registered"
|
|
203
|
+
if entry["contract_digest"] != fn.digest:
|
|
204
|
+
return None, "lockfile_stale"
|
|
205
|
+
return entry["function_id"], None
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _error(response):
|
|
209
|
+
try:
|
|
210
|
+
body = response.json()
|
|
211
|
+
code = body.get("error", "service_error")
|
|
212
|
+
if isinstance(code, dict):
|
|
213
|
+
code = code.get("code")
|
|
214
|
+
return safe_code(code)
|
|
215
|
+
except (ValueError, AttributeError):
|
|
216
|
+
return "service_error"
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _raise_response(response):
|
|
220
|
+
code, status = _error(response), response.status_code
|
|
221
|
+
kind = (
|
|
222
|
+
SeldonAuthError
|
|
223
|
+
if status in (401, 403)
|
|
224
|
+
else SeldonSpendError
|
|
225
|
+
if status == 402
|
|
226
|
+
else SeldonContractError
|
|
227
|
+
if status in (400, 422)
|
|
228
|
+
else SeldonError
|
|
229
|
+
)
|
|
230
|
+
raise kind(code, status=status)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
async def invoke(fn, inputs, *, async_mode):
|
|
234
|
+
started = time.monotonic()
|
|
235
|
+
info = CallInfo()
|
|
236
|
+
options = options_for(fn)
|
|
237
|
+
admitted = False
|
|
238
|
+
client = _client(async_mode, options)
|
|
239
|
+
|
|
240
|
+
async def request(url, body, key, timeout, headers=None):
|
|
241
|
+
kwargs = {
|
|
242
|
+
"json": body,
|
|
243
|
+
"headers": {
|
|
244
|
+
"Authorization": "Bearer " + key,
|
|
245
|
+
"User-Agent": f"seldon-sdk/{__version__} python/{platform.python_version()}",
|
|
246
|
+
**(headers or {}),
|
|
247
|
+
},
|
|
248
|
+
"timeout": timeout,
|
|
249
|
+
"follow_redirects": False,
|
|
250
|
+
}
|
|
251
|
+
result = client.post(url + "/chat/completions", **kwargs)
|
|
252
|
+
return await result if inspect.isawaitable(result) else result
|
|
253
|
+
|
|
254
|
+
async def reference(reason, cause=None):
|
|
255
|
+
info.path, info.reason = "reference", reason
|
|
256
|
+
if options["fallback"] == "never":
|
|
257
|
+
raise cause or SeldonReferenceError(reason)
|
|
258
|
+
origin = _url(options.get("reference_base_url", options["base_url"]))
|
|
259
|
+
different = urlsplit(origin).netloc != urlsplit(options["base_url"]).netloc
|
|
260
|
+
key = options.get("reference_api_key") or (None if different else options["api_key"])
|
|
261
|
+
if not key:
|
|
262
|
+
raise SeldonConfigError("reference_api_key_required")
|
|
263
|
+
body = {
|
|
264
|
+
key: fn.contract[key]
|
|
265
|
+
for key in ("model", "temperature", "max_tokens", "response_format")
|
|
266
|
+
}
|
|
267
|
+
body["messages"] = render(fn.contract, inputs)
|
|
268
|
+
try:
|
|
269
|
+
response = await request(origin, body, key, options["reference_timeout"])
|
|
270
|
+
if response.status_code != 200:
|
|
271
|
+
_raise_response(response)
|
|
272
|
+
data = response.json()
|
|
273
|
+
content = data["choices"][0]["message"]["content"]
|
|
274
|
+
if content.strip().startswith("```json") and content.strip().endswith("```"):
|
|
275
|
+
content = content.strip()[7:-3].strip()
|
|
276
|
+
info.reference_output_fenced = True
|
|
277
|
+
info.usage = data.get("usage")
|
|
278
|
+
return fn.output.validate_json(content)
|
|
279
|
+
except (httpx.HTTPError, ValueError, TypeError, KeyError, IndexError, ValidationError):
|
|
280
|
+
raise SeldonReferenceError("reference_output_invalid") from cause
|
|
281
|
+
|
|
282
|
+
try:
|
|
283
|
+
identity, reason = resolve_identity(fn)
|
|
284
|
+
info.function_id = identity
|
|
285
|
+
if os.environ.get("SELDON_MODE") == "reference":
|
|
286
|
+
return await reference("reference_mode"), info
|
|
287
|
+
if reason:
|
|
288
|
+
warn_once(fn.key, reason + ": run seldon sync")
|
|
289
|
+
return await reference(reason), info
|
|
290
|
+
if options["breaker"]:
|
|
291
|
+
admitted = fn.breaker.allow()
|
|
292
|
+
if not admitted:
|
|
293
|
+
return await reference("circuit_open"), info
|
|
294
|
+
headers = {"Idempotency-Key": str(uuid4()), "X-Seldon-Contract-Digest": fn.digest}
|
|
295
|
+
body = {
|
|
296
|
+
"model": "seldon/function/" + identity,
|
|
297
|
+
"messages": [{"role": "user", "content": canonical(inputs)}],
|
|
298
|
+
}
|
|
299
|
+
for attempt in range(3):
|
|
300
|
+
info.attempts = attempt + 1
|
|
301
|
+
try:
|
|
302
|
+
response = await request(
|
|
303
|
+
options["base_url"], body, options["api_key"], options["timeout"], headers
|
|
304
|
+
)
|
|
305
|
+
except (httpx.ConnectError, httpx.ReadTimeout):
|
|
306
|
+
if attempt < 2:
|
|
307
|
+
await asyncio.sleep(0.1 * (2**attempt))
|
|
308
|
+
continue
|
|
309
|
+
return await reference(
|
|
310
|
+
"transport_unavailable", SeldonError("transport_unavailable")
|
|
311
|
+
), info
|
|
312
|
+
status, code = response.status_code, _error(response)
|
|
313
|
+
if status == 200:
|
|
314
|
+
fn.breaker.record(False, probe=admitted == "probe")
|
|
315
|
+
try:
|
|
316
|
+
data = response.json()
|
|
317
|
+
except ValueError:
|
|
318
|
+
raise SeldonOutputValidationError("invalid_function_output") from None
|
|
319
|
+
meta = data.get("seldon", {})
|
|
320
|
+
info.revision_id, info.execution_id = (
|
|
321
|
+
meta.get("revision_id"),
|
|
322
|
+
meta.get("execution_id"),
|
|
323
|
+
)
|
|
324
|
+
info.server_fallback_used = bool(meta.get("fallback_used"))
|
|
325
|
+
info.contract_check = response.headers.get("X-Seldon-Contract-Check", "unsupported")
|
|
326
|
+
info.usage = data.get("usage")
|
|
327
|
+
try:
|
|
328
|
+
result = fn.output.validate_json(data["choices"][0]["message"]["content"])
|
|
329
|
+
except (ValueError, ValidationError, KeyError, IndexError, TypeError):
|
|
330
|
+
if options["on_invalid_output"] == "reference":
|
|
331
|
+
return await reference("invalid_function_output"), info
|
|
332
|
+
raise SeldonOutputValidationError("invalid_function_output") from None
|
|
333
|
+
return result, info
|
|
334
|
+
if status in (401, 402, 403) or status == 429 and code == "api_key_rate_limit":
|
|
335
|
+
_raise_response(response)
|
|
336
|
+
if status == 400 or status == 422:
|
|
337
|
+
_raise_response(response)
|
|
338
|
+
if status == 409 and code == "contract_digest_mismatch":
|
|
339
|
+
warn_once(fn.key, "contract_digest_mismatch: run seldon sync")
|
|
340
|
+
if options["on_drift"] == "raise":
|
|
341
|
+
raise SeldonContractError(code)
|
|
342
|
+
return await reference(code), info
|
|
343
|
+
if status in (502, 504):
|
|
344
|
+
fn.breaker.record(True, probe=admitted == "probe")
|
|
345
|
+
return await reference(
|
|
346
|
+
"server_execution_failed" if status == 502 else "server_deadline_exceeded"
|
|
347
|
+
), info
|
|
348
|
+
if (status == 429 and code == "execution_capacity") or (
|
|
349
|
+
status == 409 and code == "execution_in_progress"
|
|
350
|
+
):
|
|
351
|
+
if attempt == 2:
|
|
352
|
+
return await reference(code), info
|
|
353
|
+
try:
|
|
354
|
+
pause = float(response.headers.get("Retry-After", "0.1"))
|
|
355
|
+
pause = min(5, max(0, pause)) if math.isfinite(pause) else 0.1
|
|
356
|
+
except ValueError:
|
|
357
|
+
pause = 0.1
|
|
358
|
+
await asyncio.sleep(pause)
|
|
359
|
+
continue
|
|
360
|
+
if (status == 404 and code in {"function_not_found", "revision_not_available"}) or (
|
|
361
|
+
status == 409 and code in {"function_disabled", "execution_disabled"}
|
|
362
|
+
):
|
|
363
|
+
return await reference(code), info
|
|
364
|
+
_raise_response(response)
|
|
365
|
+
raise SeldonError("execution_unavailable")
|
|
366
|
+
finally:
|
|
367
|
+
if admitted == "probe":
|
|
368
|
+
fn.breaker.release_probe()
|
|
369
|
+
info.latency_ms = (time.monotonic() - started) * 1000
|
|
370
|
+
hook = options.get("on_call")
|
|
371
|
+
if hook:
|
|
372
|
+
try:
|
|
373
|
+
hook(info)
|
|
374
|
+
except Exception:
|
|
375
|
+
logging.getLogger(__name__).warning("call_metadata_hook_failed")
|