nativecluster 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nativecluster/__init__.py +139 -0
- nativecluster/_firings.py +185 -0
- nativecluster/_gate.py +312 -0
- nativecluster/_host.py +337 -0
- nativecluster/_hostsock.py +105 -0
- nativecluster/_launch.py +203 -0
- nativecluster/_models.py +639 -0
- nativecluster/_origin.py +217 -0
- nativecluster/_serve.py +227 -0
- nativecluster/_ticket.py +209 -0
- nativecluster/transport.py +576 -0
- nativecluster-0.1.0.dist-info/METADATA +107 -0
- nativecluster-0.1.0.dist-info/RECORD +16 -0
- nativecluster-0.1.0.dist-info/WHEEL +5 -0
- nativecluster-0.1.0.dist-info/licenses/LICENSE +202 -0
- nativecluster-0.1.0.dist-info/top_level.txt +1 -0
nativecluster/_models.py
ADDED
|
@@ -0,0 +1,639 @@
|
|
|
1
|
+
"""Models, through the host (spec/hostapi.v1.md, "Which models may do a
|
|
2
|
+
task" and "The waiting line"; the Go kit's sdk/go/app/models.go).
|
|
3
|
+
|
|
4
|
+
Every model call waits its turn in the host's one line on the origin of
|
|
5
|
+
whatever started it, runs on the grant the line gave it, and gives the grant
|
|
6
|
+
back with what it did: counts, never words. The app names one of its
|
|
7
|
+
manifest's tasks, never a model: the host says which of the group's models
|
|
8
|
+
may do it, so a better model reaches the app the day it passes, with no
|
|
9
|
+
update. The app never reaches the engine, and never names the person a call
|
|
10
|
+
is for: the origin does.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
import socket
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
import urllib.parse
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from typing import Any, Callable, List, Optional, Tuple
|
|
23
|
+
|
|
24
|
+
from ._hostsock import HostError, HostSocket, close
|
|
25
|
+
from ._origin import Origin, current_away, current_conversation, current_origin
|
|
26
|
+
|
|
27
|
+
# Kinds of model call, as the models route reads them.
|
|
28
|
+
KIND_CHAT = "chat"
|
|
29
|
+
KIND_EMBEDDING = "embedding"
|
|
30
|
+
KIND_TRANSCRIPTION = "transcription"
|
|
31
|
+
KIND_TTS = "tts"
|
|
32
|
+
KIND_IMAGE = "image"
|
|
33
|
+
|
|
34
|
+
# ROUTES is the engine route each kind of model call uses
|
|
35
|
+
# (catalog/kindroutes.go), with the kind the models route reads for it.
|
|
36
|
+
ROUTES = {
|
|
37
|
+
"/v1/chat/completions": KIND_CHAT,
|
|
38
|
+
"/v1/embeddings": KIND_EMBEDDING,
|
|
39
|
+
"/v1/audio/transcriptions": KIND_TRANSCRIPTION,
|
|
40
|
+
"/v1/audio/speech": KIND_TTS,
|
|
41
|
+
"/v1/images/generations": KIND_IMAGE,
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
# GRANT_HEADER names, on a model call, the grant the host holds for it.
|
|
45
|
+
GRANT_HEADER = "X-NC-Grant"
|
|
46
|
+
# STOPPED_HEADER and LOST_HEADER carry the host's word that a call's grant
|
|
47
|
+
# was stopped to make way for someone, or lost with the Mac it ran on: the
|
|
48
|
+
# call is run again, never read as a failure.
|
|
49
|
+
STOPPED_HEADER = "X-NC-Stopped"
|
|
50
|
+
LOST_HEADER = "X-NC-Lost"
|
|
51
|
+
|
|
52
|
+
# MAX_TRIES is how often a call stopped before its answer began is run: the
|
|
53
|
+
# line stops one call at most twice.
|
|
54
|
+
MAX_TRIES = 4
|
|
55
|
+
|
|
56
|
+
# CONTEXT_LENGTH_EXCEEDED is OpenAI's code for a prompt longer than the
|
|
57
|
+
# model's window: an agent built on OpenAI's shapes shortens its
|
|
58
|
+
# conversation when it reads it, so the kit says it in those words.
|
|
59
|
+
CONTEXT_LENGTH_EXCEEDED = "context_length_exceeded"
|
|
60
|
+
|
|
61
|
+
# The engine's words for a prompt that does not fit its window: a call
|
|
62
|
+
# answered at once says the prompt is so many tokens "but the context holds"
|
|
63
|
+
# so many; a stream that runs out of room ends with "Context size exceeded".
|
|
64
|
+
# They are the engine's, not the contract's (spec/hostapi.v1.md, "A chat
|
|
65
|
+
# call's window"), so they are read here, in one place.
|
|
66
|
+
_OVERFLOW_WORDS = ("the context holds", "context size exceeded")
|
|
67
|
+
|
|
68
|
+
_MODELS_KEPT = 60.0
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class NoOrigin(Exception):
|
|
72
|
+
def __init__(self) -> None:
|
|
73
|
+
super().__init__(
|
|
74
|
+
"nativecluster: a model call needs a person's request, a firing of the app's schedules, or a job one of "
|
|
75
|
+
"them started (call it while handling the request, or inside acting_for)"
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class NoModel(Exception):
|
|
80
|
+
"""No model on this Mac may do the task now: the install check says
|
|
81
|
+
why, and the host's Models page names one to get."""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class Refused(Exception):
|
|
85
|
+
"""The line refused the wait (its words), or it was given up. words is
|
|
86
|
+
the line's own, for the person, when it gave any."""
|
|
87
|
+
|
|
88
|
+
def __init__(self, message: str, words: str = "") -> None:
|
|
89
|
+
super().__init__(message)
|
|
90
|
+
self.words = words
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class Stopped(Exception):
|
|
94
|
+
"""The line stopped the call's answer to make way for someone, after
|
|
95
|
+
part of it had reached the app, or the answer ended early: run the turn
|
|
96
|
+
again later."""
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class ModelError(Exception):
|
|
100
|
+
"""The model's call failed: the status and the error it answered."""
|
|
101
|
+
|
|
102
|
+
def __init__(self, status: int, error: Any) -> None:
|
|
103
|
+
message = error.get("message") if isinstance(error, dict) else error
|
|
104
|
+
super().__init__("nativecluster: the model answered %d: %s" % (status, str(message)[:512]))
|
|
105
|
+
self.status = status
|
|
106
|
+
self.error = error
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class ContextTooLong(ModelError):
|
|
110
|
+
"""The prompt and the longest answer do not fit the model's window:
|
|
111
|
+
shorten the prompt (the models route's windows say how long it may be)."""
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@dataclass(frozen=True)
|
|
115
|
+
class Grant:
|
|
116
|
+
"""One turn in line: its id, the cache key its chat call carries, and
|
|
117
|
+
the model the host seated it for with the window it holds it at (empty
|
|
118
|
+
from a host that names none)."""
|
|
119
|
+
|
|
120
|
+
id: str
|
|
121
|
+
cache_key: str = ""
|
|
122
|
+
model: str = ""
|
|
123
|
+
window: int = 0
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@dataclass
|
|
127
|
+
class Answer:
|
|
128
|
+
"""A chat call's answer: its text, the model that wrote it, and the
|
|
129
|
+
prompt's and the answer's tokens."""
|
|
130
|
+
|
|
131
|
+
text: str = ""
|
|
132
|
+
model: str = ""
|
|
133
|
+
read: int = 0
|
|
134
|
+
written: int = 0
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def stopped(headers: Any) -> bool:
|
|
138
|
+
"""Whether an answer's headers say the line stopped or lost its call."""
|
|
139
|
+
return bool(headers.get(STOPPED_HEADER) or headers.get(LOST_HEADER))
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def overflow_error(error: Any) -> Optional[dict]:
|
|
143
|
+
"""error, in OpenAI's words, when it is the engine saying a prompt does
|
|
144
|
+
not fit its window; None for any other error."""
|
|
145
|
+
code = error.get("code") if isinstance(error, dict) else None
|
|
146
|
+
message = error.get("message") if isinstance(error, dict) else error
|
|
147
|
+
if code == CONTEXT_LENGTH_EXCEEDED or not isinstance(message, str):
|
|
148
|
+
return None
|
|
149
|
+
if not any(w in message.lower() for w in _OVERFLOW_WORDS):
|
|
150
|
+
return None
|
|
151
|
+
return {"message": "context length exceeded: " + message, "type": "invalid_request_error",
|
|
152
|
+
"param": "messages", "code": CONTEXT_LENGTH_EXCEEDED}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def overflow_body(status: int, body: bytes) -> bytes:
|
|
156
|
+
"""A refused call's body, an over-long prompt's said in OpenAI's words."""
|
|
157
|
+
if status != 400:
|
|
158
|
+
return body
|
|
159
|
+
try:
|
|
160
|
+
obj = json.loads(body)
|
|
161
|
+
except ValueError:
|
|
162
|
+
return body
|
|
163
|
+
if not isinstance(obj, dict):
|
|
164
|
+
return body
|
|
165
|
+
mapped = overflow_error(obj.get("error"))
|
|
166
|
+
if mapped is None:
|
|
167
|
+
return body
|
|
168
|
+
return json.dumps({"error": mapped}).encode("utf-8")
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class EventScan:
|
|
172
|
+
"""Reads an OpenAI-shaped answer stream as it passes, line by line: its
|
|
173
|
+
usage, whether it reached [DONE], the error a frame carries (an
|
|
174
|
+
over-long prompt's rewritten in OpenAI's words), and each delta's text.
|
|
175
|
+
feed returns the bytes to pass on, whole lines only, and the text."""
|
|
176
|
+
|
|
177
|
+
def __init__(self) -> None:
|
|
178
|
+
self._tail = b""
|
|
179
|
+
self.done = False
|
|
180
|
+
self.began = False
|
|
181
|
+
self.read = 0
|
|
182
|
+
self.written = 0
|
|
183
|
+
self.error: Any = None
|
|
184
|
+
|
|
185
|
+
def feed(self, chunk: bytes) -> Tuple[bytes, List[str]]:
|
|
186
|
+
lines = (self._tail + chunk).split(b"\n")
|
|
187
|
+
self._tail = lines.pop()
|
|
188
|
+
texts: List[str] = []
|
|
189
|
+
return b"".join(self._line(line, texts) + b"\n" for line in lines), texts
|
|
190
|
+
|
|
191
|
+
def flush(self) -> Tuple[bytes, List[str]]:
|
|
192
|
+
"""What is left once the stream ended: a last line with no newline."""
|
|
193
|
+
tail, self._tail = self._tail, b""
|
|
194
|
+
texts: List[str] = []
|
|
195
|
+
if not tail:
|
|
196
|
+
return b"", texts
|
|
197
|
+
return self._line(tail, texts), texts
|
|
198
|
+
|
|
199
|
+
def _line(self, line: bytes, texts: List[str]) -> bytes:
|
|
200
|
+
s = line.strip()
|
|
201
|
+
if not s.startswith(b"data:"):
|
|
202
|
+
return line
|
|
203
|
+
data = s[5:].strip()
|
|
204
|
+
if data == b"[DONE]":
|
|
205
|
+
self.done = True
|
|
206
|
+
return line
|
|
207
|
+
try:
|
|
208
|
+
obj = json.loads(data)
|
|
209
|
+
except ValueError:
|
|
210
|
+
return line
|
|
211
|
+
if not isinstance(obj, dict):
|
|
212
|
+
return line
|
|
213
|
+
err = obj.get("error")
|
|
214
|
+
if err:
|
|
215
|
+
mapped = overflow_error(err)
|
|
216
|
+
self.error = mapped or err
|
|
217
|
+
if mapped is not None:
|
|
218
|
+
return b"data: " + json.dumps({"error": mapped}).encode("utf-8")
|
|
219
|
+
return line
|
|
220
|
+
usage = obj.get("usage")
|
|
221
|
+
if isinstance(usage, dict):
|
|
222
|
+
self.read = _int(usage.get("prompt_tokens"))
|
|
223
|
+
self.written = _int(usage.get("completion_tokens"))
|
|
224
|
+
for choice in obj.get("choices") or []:
|
|
225
|
+
delta = choice.get("delta") if isinstance(choice, dict) else None
|
|
226
|
+
if not isinstance(delta, dict):
|
|
227
|
+
continue
|
|
228
|
+
if delta.get("tool_calls"):
|
|
229
|
+
self.began = True
|
|
230
|
+
content = delta.get("content")
|
|
231
|
+
if isinstance(content, str) and content:
|
|
232
|
+
self.began = True
|
|
233
|
+
texts.append(content)
|
|
234
|
+
return line
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _int(v: Any) -> int:
|
|
238
|
+
return v if isinstance(v, int) and not isinstance(v, bool) else 0
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def usage_of(body: bytes) -> Tuple[int, int]:
|
|
242
|
+
"""A whole answer's prompt and answer tokens (0 where it names none)."""
|
|
243
|
+
try:
|
|
244
|
+
usage = json.loads(body).get("usage") or {}
|
|
245
|
+
return _int(usage.get("prompt_tokens")), _int(usage.get("completion_tokens"))
|
|
246
|
+
except (ValueError, AttributeError):
|
|
247
|
+
return 0, 0
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
_PART_NAME = re.compile(rb'name="([^"]*)"')
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def multipart_field(body: bytes, content_type: str, name: str, value: Optional[str] = None) -> Tuple[bytes, Optional[str]]:
|
|
254
|
+
"""A multipart/form-data body's field name: its value, and the body with
|
|
255
|
+
it set to value (when given). A body without the field is unchanged."""
|
|
256
|
+
m = re.search(r'boundary="?([^";]+)"?', content_type)
|
|
257
|
+
if not m:
|
|
258
|
+
return body, None
|
|
259
|
+
sep = b"--" + m.group(1).encode("latin-1")
|
|
260
|
+
parts = body.split(sep)
|
|
261
|
+
for i, part in enumerate(parts):
|
|
262
|
+
head, blank, rest = part.partition(b"\r\n\r\n")
|
|
263
|
+
if not blank:
|
|
264
|
+
continue
|
|
265
|
+
found = _PART_NAME.search(head)
|
|
266
|
+
if not found or found.group(1).decode("latin-1") != name or b"filename=" in head:
|
|
267
|
+
continue
|
|
268
|
+
end = rest.rfind(b"\r\n")
|
|
269
|
+
old = (rest[:end] if end >= 0 else rest).decode("utf-8", errors="replace")
|
|
270
|
+
if value is not None:
|
|
271
|
+
parts[i] = head + blank + value.encode("utf-8") + (rest[end:] if end >= 0 else b"")
|
|
272
|
+
return sep.join(parts), old
|
|
273
|
+
return body, None
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
class Waiting:
|
|
277
|
+
"""A wait in line another thread may give up: cancel closes it, and the
|
|
278
|
+
host then gives the place in line away. A grant that arrives as it is
|
|
279
|
+
given up is handed to whoever gave it up, to give back, never lost."""
|
|
280
|
+
|
|
281
|
+
def __init__(self) -> None:
|
|
282
|
+
self._lock = threading.Lock()
|
|
283
|
+
self._sock: Optional[socket.socket] = None
|
|
284
|
+
self._grant: Optional[Grant] = None
|
|
285
|
+
self._taken = False
|
|
286
|
+
self.cancelled = False
|
|
287
|
+
|
|
288
|
+
def _attach(self, sock: socket.socket) -> None:
|
|
289
|
+
with self._lock:
|
|
290
|
+
if self.cancelled:
|
|
291
|
+
raise Refused("nativecluster: the wait was given up")
|
|
292
|
+
self._sock = sock
|
|
293
|
+
|
|
294
|
+
def _deliver(self, grant: Grant) -> bool:
|
|
295
|
+
with self._lock:
|
|
296
|
+
if self.cancelled:
|
|
297
|
+
return False
|
|
298
|
+
self._grant = grant
|
|
299
|
+
return True
|
|
300
|
+
|
|
301
|
+
def take(self) -> Optional[Grant]:
|
|
302
|
+
"""The grant the wait got, now the caller's."""
|
|
303
|
+
with self._lock:
|
|
304
|
+
self._taken = True
|
|
305
|
+
return self._grant
|
|
306
|
+
|
|
307
|
+
def cancel(self) -> Optional[Grant]:
|
|
308
|
+
"""Gives the wait up: the grant it already got and nobody took, to
|
|
309
|
+
give back, or None."""
|
|
310
|
+
with self._lock:
|
|
311
|
+
self.cancelled = True
|
|
312
|
+
sock = self._sock
|
|
313
|
+
grant = None if self._taken else self._grant
|
|
314
|
+
self._grant = None
|
|
315
|
+
if sock is not None:
|
|
316
|
+
try:
|
|
317
|
+
sock.shutdown(socket.SHUT_RDWR)
|
|
318
|
+
except OSError:
|
|
319
|
+
pass
|
|
320
|
+
return grant
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
class Models:
|
|
324
|
+
"""An app's way to its group's models, over the host's socket."""
|
|
325
|
+
|
|
326
|
+
def __init__(self, host: HostSocket, test: Optional[dict] = None) -> None:
|
|
327
|
+
self.host = host
|
|
328
|
+
self._test = test or None
|
|
329
|
+
self._lock = threading.Lock()
|
|
330
|
+
self._answers: dict = {}
|
|
331
|
+
|
|
332
|
+
# --- which model ---
|
|
333
|
+
|
|
334
|
+
def answer(self, task: str, kind: str = KIND_CHAT, lang: str = "") -> dict:
|
|
335
|
+
"""The models route's whole answer for task: models (best first),
|
|
336
|
+
to_get, and for chat each model's window (windows); kept a minute."""
|
|
337
|
+
key = (task, kind, lang)
|
|
338
|
+
now = time.monotonic()
|
|
339
|
+
with self._lock:
|
|
340
|
+
kept = self._answers.get(key)
|
|
341
|
+
if kept and now - kept[0] < _MODELS_KEPT:
|
|
342
|
+
return kept[1]
|
|
343
|
+
q = {"task": task, "kind": kind}
|
|
344
|
+
if lang:
|
|
345
|
+
q["lang"] = lang
|
|
346
|
+
out = self.host.call("GET", "/host/v1/models?" + urllib.parse.urlencode(q)) or {}
|
|
347
|
+
with self._lock:
|
|
348
|
+
self._answers[key] = (now, out)
|
|
349
|
+
return out
|
|
350
|
+
|
|
351
|
+
def choices(self, task: str, kind: str = KIND_CHAT) -> List[str]:
|
|
352
|
+
"""The models on this Mac that may do task for a call of kind, best
|
|
353
|
+
first. Empty: none may now (an optional task's feature stays hidden)."""
|
|
354
|
+
if self._test and self._test.get("model"):
|
|
355
|
+
return [self._test["model"]]
|
|
356
|
+
return [m for m in self.answer(task, kind).get("models") or [] if isinstance(m, str)]
|
|
357
|
+
|
|
358
|
+
def window(self, task: str, model: str = "") -> int:
|
|
359
|
+
"""The window, in tokens for the prompt and the answer together, the
|
|
360
|
+
host runs model's chat calls in (the task's first model when none is
|
|
361
|
+
named); 0 when the host names none. Size every prompt from it."""
|
|
362
|
+
if not model:
|
|
363
|
+
ms = self.choices(task)
|
|
364
|
+
model = ms[0] if ms else ""
|
|
365
|
+
w = (self.answer(task).get("windows") or {}).get(model)
|
|
366
|
+
return w if isinstance(w, int) else 0
|
|
367
|
+
|
|
368
|
+
def model_for(self, task: str, kind: str = KIND_CHAT, want: str = "") -> str:
|
|
369
|
+
"""The model to call for task: want, a person's pick, while the host
|
|
370
|
+
names it for the task; else the task's first."""
|
|
371
|
+
ms = self.choices(task, kind)
|
|
372
|
+
if not ms:
|
|
373
|
+
raise NoModel("nativecluster: no model on this Mac may do %s yet" % task)
|
|
374
|
+
return want if want in ms else ms[0]
|
|
375
|
+
|
|
376
|
+
def seats_for(self, model: str) -> int:
|
|
377
|
+
"""How many of a model's calls may run at once across the group's
|
|
378
|
+
Macs: run about that many independent units, never more."""
|
|
379
|
+
state = self.host.call("GET", "/host/v1/line/state") or {}
|
|
380
|
+
n = (state.get("seats_for") or {}).get(model)
|
|
381
|
+
return n if isinstance(n, int) and n > 0 else 1
|
|
382
|
+
|
|
383
|
+
# --- a turn in line ---
|
|
384
|
+
|
|
385
|
+
def origin(self, origin: Optional[Origin] = None) -> Origin:
|
|
386
|
+
o = origin or current_origin()
|
|
387
|
+
if o is not None:
|
|
388
|
+
return o
|
|
389
|
+
t = (self._test or {}).get("ticket") or {}
|
|
390
|
+
if t.get("origin") == "request":
|
|
391
|
+
return Origin("request", person=t.get("person", ""), request=t.get("request", ""))
|
|
392
|
+
raise NoOrigin()
|
|
393
|
+
|
|
394
|
+
def wait(self, task: str, *, model: str = "", pick_chat: bool = False, origin: Optional[Origin] = None,
|
|
395
|
+
conversation: Optional[str] = None, waiting: Optional[Waiting] = None,
|
|
396
|
+
on_position: Optional[Callable[[int], None]] = None,
|
|
397
|
+
on_away: Optional[Callable[[str], None]] = None) -> Grant:
|
|
398
|
+
"""Waits in the host's line for one call's seat, as origin. pick_chat
|
|
399
|
+
says the call is one chat call: the host may seat it on any of the
|
|
400
|
+
group's Macs holding the model, and names the model it seated it for.
|
|
401
|
+
on_away (else when_away's) hears the host's words when the wait is
|
|
402
|
+
held because the cluster is out of reach; the wait goes on."""
|
|
403
|
+
o = self.origin(origin)
|
|
404
|
+
if on_away is None:
|
|
405
|
+
on_away = current_away()
|
|
406
|
+
tag: dict = {"Person": o.person, "Task": task}
|
|
407
|
+
if model:
|
|
408
|
+
tag["Model"] = model
|
|
409
|
+
if o.job:
|
|
410
|
+
tag["Job"] = True
|
|
411
|
+
conv = current_conversation() if conversation is None else conversation
|
|
412
|
+
if conv:
|
|
413
|
+
tag["Conversation"] = conv
|
|
414
|
+
body: dict = {"kind": "acquire", "lane": o.lane, "tag": tag, "ticket": o.wire()}
|
|
415
|
+
if pick_chat:
|
|
416
|
+
body["pick"] = "chat"
|
|
417
|
+
conn, resp = self.host.open("POST", "/host/v1/line/wait", json.dumps(body).encode("utf-8"),
|
|
418
|
+
{"Content-Type": "application/json"}, timeout=None,
|
|
419
|
+
sent=waiting._attach if waiting is not None else None)
|
|
420
|
+
try:
|
|
421
|
+
if resp.status != 200:
|
|
422
|
+
raise HostError("POST", "/host/v1/line/wait", resp.status, resp.read(512).decode("utf-8", "replace"))
|
|
423
|
+
while True:
|
|
424
|
+
line = resp.readline(1 << 20)
|
|
425
|
+
if not line and waiting is not None and waiting.cancelled:
|
|
426
|
+
raise Refused("nativecluster: the wait was given up")
|
|
427
|
+
if not line:
|
|
428
|
+
raise HostError("POST", "/host/v1/line/wait", 200, "the line ended the wait without an answer")
|
|
429
|
+
if not line.strip():
|
|
430
|
+
continue
|
|
431
|
+
msg = json.loads(line)
|
|
432
|
+
if msg.get("refused"):
|
|
433
|
+
raise Refused("nativecluster: the host's line refused the call: %s" % msg["refused"], msg["refused"])
|
|
434
|
+
if msg.get("id"):
|
|
435
|
+
g = Grant(id=msg["id"], cache_key=msg.get("cache_key") or "", model=msg.get("model") or "",
|
|
436
|
+
window=_int(msg.get("window")))
|
|
437
|
+
if waiting is not None and not waiting._deliver(g):
|
|
438
|
+
self.release(g, model)
|
|
439
|
+
raise Refused("nativecluster: the wait was given up")
|
|
440
|
+
return g
|
|
441
|
+
if on_position is not None and _int(msg.get("position")) > 0:
|
|
442
|
+
on_position(msg["position"])
|
|
443
|
+
if on_away is not None and isinstance(msg.get("away"), str) and msg["away"]:
|
|
444
|
+
on_away(msg["away"])
|
|
445
|
+
except OSError:
|
|
446
|
+
if waiting is not None and waiting.cancelled:
|
|
447
|
+
raise Refused("nativecluster: the wait was given up") from None
|
|
448
|
+
raise
|
|
449
|
+
finally:
|
|
450
|
+
close(conn, resp)
|
|
451
|
+
|
|
452
|
+
def release(self, grant: Grant, model: str = "", read: int = 0, written: int = 0) -> None:
|
|
453
|
+
"""Gives a grant back with what its call did: counts, never words."""
|
|
454
|
+
try:
|
|
455
|
+
self.host.call("POST", "/host/v1/line/release", {
|
|
456
|
+
"grant": grant.id,
|
|
457
|
+
"notes": {"model": model or grant.model, "calls": 1, "read": read, "written": written},
|
|
458
|
+
})
|
|
459
|
+
except (OSError, HostError):
|
|
460
|
+
pass # the host ends the grant with the app's process at the latest
|
|
461
|
+
|
|
462
|
+
def give_back(self, grant: Grant) -> None:
|
|
463
|
+
"""Gives back a grant no call used (a wait given up as it was
|
|
464
|
+
answered): nothing to count."""
|
|
465
|
+
try:
|
|
466
|
+
self.host.call("POST", "/host/v1/line/release", {"grant": grant.id})
|
|
467
|
+
except (OSError, HostError):
|
|
468
|
+
pass # the host ends the grant with the app's process at the latest
|
|
469
|
+
|
|
470
|
+
def post(self, grant: Grant, path: str, content_type: str, body: bytes):
|
|
471
|
+
"""Sends one model call on grant: its connection and unread answer."""
|
|
472
|
+
return self.host.open("POST", path, body, {GRANT_HEADER: grant.id, "Content-Type": content_type}, timeout=None)
|
|
473
|
+
|
|
474
|
+
# --- calls ---
|
|
475
|
+
|
|
476
|
+
def _chat_body(self, model: str, grant: Grant, messages: list, max_tokens: int, extra: dict, stream: bool) -> bytes:
|
|
477
|
+
b = dict(extra)
|
|
478
|
+
b["model"] = model
|
|
479
|
+
b["messages"] = messages
|
|
480
|
+
b["stream"] = stream
|
|
481
|
+
if max_tokens > 0:
|
|
482
|
+
b["max_tokens"] = max_tokens
|
|
483
|
+
if stream:
|
|
484
|
+
b["stream_options"] = {"include_usage": True}
|
|
485
|
+
if grant.cache_key:
|
|
486
|
+
b["cache_key"] = grant.cache_key
|
|
487
|
+
return json.dumps(b).encode("utf-8")
|
|
488
|
+
|
|
489
|
+
def chat(self, task: str, messages: list, *, max_tokens: int = 0, origin: Optional[Origin] = None, **extra: Any) -> Answer:
|
|
490
|
+
"""A chat model's whole answer for task. extra carries any other
|
|
491
|
+
OpenAI-shaped field (response_format, temperature, tools), never
|
|
492
|
+
model or stream."""
|
|
493
|
+
o = self.origin(origin)
|
|
494
|
+
asked = self.model_for(task)
|
|
495
|
+
for attempt in range(MAX_TRIES):
|
|
496
|
+
g = self.wait(task, model=asked, pick_chat=True, origin=o)
|
|
497
|
+
model = g.model or asked
|
|
498
|
+
conn, resp = self.post(g, "/v1/chat/completions", "application/json",
|
|
499
|
+
self._chat_body(model, g, messages, max_tokens, extra, False))
|
|
500
|
+
try:
|
|
501
|
+
raw = resp.read(16 << 20)
|
|
502
|
+
except BaseException:
|
|
503
|
+
self.release(g, model)
|
|
504
|
+
raise
|
|
505
|
+
finally:
|
|
506
|
+
close(conn, resp)
|
|
507
|
+
if stopped(resp.headers):
|
|
508
|
+
self.release(g, model)
|
|
509
|
+
if attempt + 1 < MAX_TRIES:
|
|
510
|
+
continue
|
|
511
|
+
raise Stopped("nativecluster: the waiting line stopped this answer to make way for someone; try again")
|
|
512
|
+
read, written = usage_of(raw)
|
|
513
|
+
self.release(g, model, read, written)
|
|
514
|
+
if resp.status != 200:
|
|
515
|
+
raise _model_error(resp.status, raw)
|
|
516
|
+
try:
|
|
517
|
+
text = json.loads(raw)["choices"][0]["message"].get("content") or ""
|
|
518
|
+
except (ValueError, KeyError, IndexError, TypeError, AttributeError):
|
|
519
|
+
raise ModelError(resp.status, "the model's answer did not read") from None
|
|
520
|
+
return Answer(text=text, model=model, read=read, written=written)
|
|
521
|
+
raise AssertionError("unreachable")
|
|
522
|
+
|
|
523
|
+
def chat_stream(self, task: str, messages: list, on_delta: Optional[Callable[[str], None]] = None, *,
|
|
524
|
+
max_tokens: int = 0, origin: Optional[Origin] = None, **extra: Any) -> Answer:
|
|
525
|
+
"""A chat model's answer for task, each piece passed to on_delta as it
|
|
526
|
+
is written. A stop before the first piece runs the call again; after
|
|
527
|
+
it, the answer ends with Stopped."""
|
|
528
|
+
o = self.origin(origin)
|
|
529
|
+
asked = self.model_for(task)
|
|
530
|
+
for attempt in range(MAX_TRIES):
|
|
531
|
+
g = self.wait(task, model=asked, pick_chat=True, origin=o)
|
|
532
|
+
model = g.model or asked
|
|
533
|
+
scan = EventScan()
|
|
534
|
+
text: List[str] = []
|
|
535
|
+
try:
|
|
536
|
+
self._stream(g, model, messages, max_tokens, extra, scan, text, on_delta)
|
|
537
|
+
except Stopped:
|
|
538
|
+
if not scan.began and attempt + 1 < MAX_TRIES:
|
|
539
|
+
continue
|
|
540
|
+
raise
|
|
541
|
+
finally:
|
|
542
|
+
self.release(g, model, scan.read, scan.written)
|
|
543
|
+
return Answer(text="".join(text), model=model, read=scan.read, written=scan.written)
|
|
544
|
+
raise AssertionError("unreachable")
|
|
545
|
+
|
|
546
|
+
def _stream(self, g: Grant, model: str, messages: list, max_tokens: int, extra: dict, scan: EventScan,
|
|
547
|
+
text: List[str], on_delta: Optional[Callable[[str], None]]) -> None:
|
|
548
|
+
conn, resp = self.post(g, "/v1/chat/completions", "application/json",
|
|
549
|
+
self._chat_body(model, g, messages, max_tokens, extra, True))
|
|
550
|
+
try:
|
|
551
|
+
if stopped(resp.headers):
|
|
552
|
+
raise Stopped("nativecluster: the waiting line stopped this answer to make way for someone; try again")
|
|
553
|
+
if resp.status != 200:
|
|
554
|
+
raise _model_error(resp.status, resp.read(64 << 10))
|
|
555
|
+
while not scan.done:
|
|
556
|
+
chunk = resp.read1(64 << 10) if hasattr(resp, "read1") else resp.read(4096)
|
|
557
|
+
if not chunk:
|
|
558
|
+
break
|
|
559
|
+
_, pieces = scan.feed(chunk)
|
|
560
|
+
self._pieces(scan, pieces, text, on_delta)
|
|
561
|
+
if not scan.done:
|
|
562
|
+
_, pieces = scan.flush()
|
|
563
|
+
self._pieces(scan, pieces, text, on_delta)
|
|
564
|
+
if scan.error is not None:
|
|
565
|
+
raise _model_error(200, json.dumps({"error": scan.error}).encode("utf-8"))
|
|
566
|
+
if not scan.done:
|
|
567
|
+
# The host says why an answer ended in a trailer this client
|
|
568
|
+
# cannot read: the line stopping it is the usual reason.
|
|
569
|
+
raise Stopped("nativecluster: the answer ended early (the waiting line may have stopped it to make way for someone); try again")
|
|
570
|
+
finally:
|
|
571
|
+
close(conn, resp)
|
|
572
|
+
|
|
573
|
+
@staticmethod
|
|
574
|
+
def _pieces(scan: EventScan, pieces: List[str], text: List[str], on_delta: Optional[Callable[[str], None]]) -> None:
|
|
575
|
+
for p in pieces:
|
|
576
|
+
text.append(p)
|
|
577
|
+
if on_delta is not None:
|
|
578
|
+
on_delta(p)
|
|
579
|
+
|
|
580
|
+
def embed(self, task: str, inputs: List[str], *, origin: Optional[Origin] = None) -> List[List[float]]:
|
|
581
|
+
"""A vector for each input, from an embedding model for task."""
|
|
582
|
+
o = self.origin(origin)
|
|
583
|
+
model = self.model_for(task, KIND_EMBEDDING)
|
|
584
|
+
raw = self._once(task, model, o, "/v1/embeddings", "application/json",
|
|
585
|
+
json.dumps({"model": model, "input": inputs}).encode("utf-8"))
|
|
586
|
+
vecs: List[List[float]] = [[] for _ in inputs]
|
|
587
|
+
for d in json.loads(raw).get("data") or []:
|
|
588
|
+
i = d.get("index")
|
|
589
|
+
if isinstance(i, int) and 0 <= i < len(vecs):
|
|
590
|
+
vecs[i] = d.get("embedding") or []
|
|
591
|
+
return vecs
|
|
592
|
+
|
|
593
|
+
def transcribe(self, task: str, audio: bytes, filename: str, *, origin: Optional[Origin] = None) -> str:
|
|
594
|
+
"""The text of a recording (a file's bytes, named so its kind shows),
|
|
595
|
+
from a speech model for task."""
|
|
596
|
+
o = self.origin(origin)
|
|
597
|
+
model = self.model_for(task, KIND_TRANSCRIPTION)
|
|
598
|
+
boundary = "nativecluster" + "%x" % int(time.time() * 1e6)
|
|
599
|
+
body = (
|
|
600
|
+
("--%s\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n%s\r\n" % (boundary, model)).encode("utf-8")
|
|
601
|
+
+ ("--%s\r\nContent-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\n"
|
|
602
|
+
"Content-Type: application/octet-stream\r\n\r\n" % (boundary, filename.replace('"', ""))).encode("utf-8")
|
|
603
|
+
+ audio + ("\r\n--%s--\r\n" % boundary).encode("utf-8")
|
|
604
|
+
)
|
|
605
|
+
raw = self._once(task, model, o, "/v1/audio/transcriptions", "multipart/form-data; boundary=" + boundary, body)
|
|
606
|
+
return json.loads(raw).get("text") or ""
|
|
607
|
+
|
|
608
|
+
def _once(self, task: str, model: str, origin: Origin, path: str, content_type: str, body: bytes) -> bytes:
|
|
609
|
+
"""One call that is not streamed, run again if the line stops it."""
|
|
610
|
+
for attempt in range(MAX_TRIES):
|
|
611
|
+
g = self.wait(task, model=model, origin=origin)
|
|
612
|
+
try:
|
|
613
|
+
conn, resp = self.post(g, path, content_type, body)
|
|
614
|
+
try:
|
|
615
|
+
raw = resp.read(64 << 20)
|
|
616
|
+
finally:
|
|
617
|
+
close(conn, resp)
|
|
618
|
+
finally:
|
|
619
|
+
self.release(g, model)
|
|
620
|
+
if stopped(resp.headers):
|
|
621
|
+
if attempt + 1 < MAX_TRIES:
|
|
622
|
+
continue
|
|
623
|
+
raise Stopped("nativecluster: the waiting line stopped this call to make way for someone; try again")
|
|
624
|
+
if resp.status != 200:
|
|
625
|
+
raise _model_error(resp.status, raw)
|
|
626
|
+
return raw
|
|
627
|
+
raise AssertionError("unreachable")
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _model_error(status: int, raw: bytes) -> ModelError:
|
|
631
|
+
try:
|
|
632
|
+
err = json.loads(raw).get("error", raw.decode("utf-8", "replace"))
|
|
633
|
+
except (ValueError, AttributeError):
|
|
634
|
+
err = raw.decode("utf-8", "replace")
|
|
635
|
+
if overflow_error(err) is not None:
|
|
636
|
+
return ContextTooLong(status, overflow_error(err))
|
|
637
|
+
if isinstance(err, dict) and err.get("code") == CONTEXT_LENGTH_EXCEEDED:
|
|
638
|
+
return ContextTooLong(status, err)
|
|
639
|
+
return ModelError(status, err)
|