nativecluster 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,639 @@
1
+ """Models, through the host (spec/hostapi.v1.md, "Which models may do a
2
+ task" and "The waiting line"; the Go kit's sdk/go/app/models.go).
3
+
4
+ Every model call waits its turn in the host's one line on the origin of
5
+ whatever started it, runs on the grant the line gave it, and gives the grant
6
+ back with what it did: counts, never words. The app names one of its
7
+ manifest's tasks, never a model: the host says which of the group's models
8
+ may do it, so a better model reaches the app the day it passes, with no
9
+ update. The app never reaches the engine, and never names the person a call
10
+ is for: the origin does.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import re
17
+ import socket
18
+ import threading
19
+ import time
20
+ import urllib.parse
21
+ from dataclasses import dataclass
22
+ from typing import Any, Callable, List, Optional, Tuple
23
+
24
+ from ._hostsock import HostError, HostSocket, close
25
+ from ._origin import Origin, current_away, current_conversation, current_origin
26
+
27
+ # Kinds of model call, as the models route reads them.
28
+ KIND_CHAT = "chat"
29
+ KIND_EMBEDDING = "embedding"
30
+ KIND_TRANSCRIPTION = "transcription"
31
+ KIND_TTS = "tts"
32
+ KIND_IMAGE = "image"
33
+
34
+ # ROUTES is the engine route each kind of model call uses
35
+ # (catalog/kindroutes.go), with the kind the models route reads for it.
36
+ ROUTES = {
37
+ "/v1/chat/completions": KIND_CHAT,
38
+ "/v1/embeddings": KIND_EMBEDDING,
39
+ "/v1/audio/transcriptions": KIND_TRANSCRIPTION,
40
+ "/v1/audio/speech": KIND_TTS,
41
+ "/v1/images/generations": KIND_IMAGE,
42
+ }
43
+
44
+ # GRANT_HEADER names, on a model call, the grant the host holds for it.
45
+ GRANT_HEADER = "X-NC-Grant"
46
+ # STOPPED_HEADER and LOST_HEADER carry the host's word that a call's grant
47
+ # was stopped to make way for someone, or lost with the Mac it ran on: the
48
+ # call is run again, never read as a failure.
49
+ STOPPED_HEADER = "X-NC-Stopped"
50
+ LOST_HEADER = "X-NC-Lost"
51
+
52
+ # MAX_TRIES is how often a call stopped before its answer began is run: the
53
+ # line stops one call at most twice.
54
+ MAX_TRIES = 4
55
+
56
+ # CONTEXT_LENGTH_EXCEEDED is OpenAI's code for a prompt longer than the
57
+ # model's window: an agent built on OpenAI's shapes shortens its
58
+ # conversation when it reads it, so the kit says it in those words.
59
+ CONTEXT_LENGTH_EXCEEDED = "context_length_exceeded"
60
+
61
+ # The engine's words for a prompt that does not fit its window: a call
62
+ # answered at once says the prompt is so many tokens "but the context holds"
63
+ # so many; a stream that runs out of room ends with "Context size exceeded".
64
+ # They are the engine's, not the contract's (spec/hostapi.v1.md, "A chat
65
+ # call's window"), so they are read here, in one place.
66
+ _OVERFLOW_WORDS = ("the context holds", "context size exceeded")
67
+
68
+ _MODELS_KEPT = 60.0
69
+
70
+
71
+ class NoOrigin(Exception):
72
+ def __init__(self) -> None:
73
+ super().__init__(
74
+ "nativecluster: a model call needs a person's request, a firing of the app's schedules, or a job one of "
75
+ "them started (call it while handling the request, or inside acting_for)"
76
+ )
77
+
78
+
79
+ class NoModel(Exception):
80
+ """No model on this Mac may do the task now: the install check says
81
+ why, and the host's Models page names one to get."""
82
+
83
+
84
+ class Refused(Exception):
85
+ """The line refused the wait (its words), or it was given up. words is
86
+ the line's own, for the person, when it gave any."""
87
+
88
+ def __init__(self, message: str, words: str = "") -> None:
89
+ super().__init__(message)
90
+ self.words = words
91
+
92
+
93
+ class Stopped(Exception):
94
+ """The line stopped the call's answer to make way for someone, after
95
+ part of it had reached the app, or the answer ended early: run the turn
96
+ again later."""
97
+
98
+
99
+ class ModelError(Exception):
100
+ """The model's call failed: the status and the error it answered."""
101
+
102
+ def __init__(self, status: int, error: Any) -> None:
103
+ message = error.get("message") if isinstance(error, dict) else error
104
+ super().__init__("nativecluster: the model answered %d: %s" % (status, str(message)[:512]))
105
+ self.status = status
106
+ self.error = error
107
+
108
+
109
+ class ContextTooLong(ModelError):
110
+ """The prompt and the longest answer do not fit the model's window:
111
+ shorten the prompt (the models route's windows say how long it may be)."""
112
+
113
+
114
+ @dataclass(frozen=True)
115
+ class Grant:
116
+ """One turn in line: its id, the cache key its chat call carries, and
117
+ the model the host seated it for with the window it holds it at (empty
118
+ from a host that names none)."""
119
+
120
+ id: str
121
+ cache_key: str = ""
122
+ model: str = ""
123
+ window: int = 0
124
+
125
+
126
+ @dataclass
127
+ class Answer:
128
+ """A chat call's answer: its text, the model that wrote it, and the
129
+ prompt's and the answer's tokens."""
130
+
131
+ text: str = ""
132
+ model: str = ""
133
+ read: int = 0
134
+ written: int = 0
135
+
136
+
137
+ def stopped(headers: Any) -> bool:
138
+ """Whether an answer's headers say the line stopped or lost its call."""
139
+ return bool(headers.get(STOPPED_HEADER) or headers.get(LOST_HEADER))
140
+
141
+
142
+ def overflow_error(error: Any) -> Optional[dict]:
143
+ """error, in OpenAI's words, when it is the engine saying a prompt does
144
+ not fit its window; None for any other error."""
145
+ code = error.get("code") if isinstance(error, dict) else None
146
+ message = error.get("message") if isinstance(error, dict) else error
147
+ if code == CONTEXT_LENGTH_EXCEEDED or not isinstance(message, str):
148
+ return None
149
+ if not any(w in message.lower() for w in _OVERFLOW_WORDS):
150
+ return None
151
+ return {"message": "context length exceeded: " + message, "type": "invalid_request_error",
152
+ "param": "messages", "code": CONTEXT_LENGTH_EXCEEDED}
153
+
154
+
155
+ def overflow_body(status: int, body: bytes) -> bytes:
156
+ """A refused call's body, an over-long prompt's said in OpenAI's words."""
157
+ if status != 400:
158
+ return body
159
+ try:
160
+ obj = json.loads(body)
161
+ except ValueError:
162
+ return body
163
+ if not isinstance(obj, dict):
164
+ return body
165
+ mapped = overflow_error(obj.get("error"))
166
+ if mapped is None:
167
+ return body
168
+ return json.dumps({"error": mapped}).encode("utf-8")
169
+
170
+
171
+ class EventScan:
172
+ """Reads an OpenAI-shaped answer stream as it passes, line by line: its
173
+ usage, whether it reached [DONE], the error a frame carries (an
174
+ over-long prompt's rewritten in OpenAI's words), and each delta's text.
175
+ feed returns the bytes to pass on, whole lines only, and the text."""
176
+
177
+ def __init__(self) -> None:
178
+ self._tail = b""
179
+ self.done = False
180
+ self.began = False
181
+ self.read = 0
182
+ self.written = 0
183
+ self.error: Any = None
184
+
185
+ def feed(self, chunk: bytes) -> Tuple[bytes, List[str]]:
186
+ lines = (self._tail + chunk).split(b"\n")
187
+ self._tail = lines.pop()
188
+ texts: List[str] = []
189
+ return b"".join(self._line(line, texts) + b"\n" for line in lines), texts
190
+
191
+ def flush(self) -> Tuple[bytes, List[str]]:
192
+ """What is left once the stream ended: a last line with no newline."""
193
+ tail, self._tail = self._tail, b""
194
+ texts: List[str] = []
195
+ if not tail:
196
+ return b"", texts
197
+ return self._line(tail, texts), texts
198
+
199
+ def _line(self, line: bytes, texts: List[str]) -> bytes:
200
+ s = line.strip()
201
+ if not s.startswith(b"data:"):
202
+ return line
203
+ data = s[5:].strip()
204
+ if data == b"[DONE]":
205
+ self.done = True
206
+ return line
207
+ try:
208
+ obj = json.loads(data)
209
+ except ValueError:
210
+ return line
211
+ if not isinstance(obj, dict):
212
+ return line
213
+ err = obj.get("error")
214
+ if err:
215
+ mapped = overflow_error(err)
216
+ self.error = mapped or err
217
+ if mapped is not None:
218
+ return b"data: " + json.dumps({"error": mapped}).encode("utf-8")
219
+ return line
220
+ usage = obj.get("usage")
221
+ if isinstance(usage, dict):
222
+ self.read = _int(usage.get("prompt_tokens"))
223
+ self.written = _int(usage.get("completion_tokens"))
224
+ for choice in obj.get("choices") or []:
225
+ delta = choice.get("delta") if isinstance(choice, dict) else None
226
+ if not isinstance(delta, dict):
227
+ continue
228
+ if delta.get("tool_calls"):
229
+ self.began = True
230
+ content = delta.get("content")
231
+ if isinstance(content, str) and content:
232
+ self.began = True
233
+ texts.append(content)
234
+ return line
235
+
236
+
237
+ def _int(v: Any) -> int:
238
+ return v if isinstance(v, int) and not isinstance(v, bool) else 0
239
+
240
+
241
+ def usage_of(body: bytes) -> Tuple[int, int]:
242
+ """A whole answer's prompt and answer tokens (0 where it names none)."""
243
+ try:
244
+ usage = json.loads(body).get("usage") or {}
245
+ return _int(usage.get("prompt_tokens")), _int(usage.get("completion_tokens"))
246
+ except (ValueError, AttributeError):
247
+ return 0, 0
248
+
249
+
250
+ _PART_NAME = re.compile(rb'name="([^"]*)"')
251
+
252
+
253
+ def multipart_field(body: bytes, content_type: str, name: str, value: Optional[str] = None) -> Tuple[bytes, Optional[str]]:
254
+ """A multipart/form-data body's field name: its value, and the body with
255
+ it set to value (when given). A body without the field is unchanged."""
256
+ m = re.search(r'boundary="?([^";]+)"?', content_type)
257
+ if not m:
258
+ return body, None
259
+ sep = b"--" + m.group(1).encode("latin-1")
260
+ parts = body.split(sep)
261
+ for i, part in enumerate(parts):
262
+ head, blank, rest = part.partition(b"\r\n\r\n")
263
+ if not blank:
264
+ continue
265
+ found = _PART_NAME.search(head)
266
+ if not found or found.group(1).decode("latin-1") != name or b"filename=" in head:
267
+ continue
268
+ end = rest.rfind(b"\r\n")
269
+ old = (rest[:end] if end >= 0 else rest).decode("utf-8", errors="replace")
270
+ if value is not None:
271
+ parts[i] = head + blank + value.encode("utf-8") + (rest[end:] if end >= 0 else b"")
272
+ return sep.join(parts), old
273
+ return body, None
274
+
275
+
276
+ class Waiting:
277
+ """A wait in line another thread may give up: cancel closes it, and the
278
+ host then gives the place in line away. A grant that arrives as it is
279
+ given up is handed to whoever gave it up, to give back, never lost."""
280
+
281
+ def __init__(self) -> None:
282
+ self._lock = threading.Lock()
283
+ self._sock: Optional[socket.socket] = None
284
+ self._grant: Optional[Grant] = None
285
+ self._taken = False
286
+ self.cancelled = False
287
+
288
+ def _attach(self, sock: socket.socket) -> None:
289
+ with self._lock:
290
+ if self.cancelled:
291
+ raise Refused("nativecluster: the wait was given up")
292
+ self._sock = sock
293
+
294
+ def _deliver(self, grant: Grant) -> bool:
295
+ with self._lock:
296
+ if self.cancelled:
297
+ return False
298
+ self._grant = grant
299
+ return True
300
+
301
+ def take(self) -> Optional[Grant]:
302
+ """The grant the wait got, now the caller's."""
303
+ with self._lock:
304
+ self._taken = True
305
+ return self._grant
306
+
307
+ def cancel(self) -> Optional[Grant]:
308
+ """Gives the wait up: the grant it already got and nobody took, to
309
+ give back, or None."""
310
+ with self._lock:
311
+ self.cancelled = True
312
+ sock = self._sock
313
+ grant = None if self._taken else self._grant
314
+ self._grant = None
315
+ if sock is not None:
316
+ try:
317
+ sock.shutdown(socket.SHUT_RDWR)
318
+ except OSError:
319
+ pass
320
+ return grant
321
+
322
+
323
+ class Models:
324
+ """An app's way to its group's models, over the host's socket."""
325
+
326
+ def __init__(self, host: HostSocket, test: Optional[dict] = None) -> None:
327
+ self.host = host
328
+ self._test = test or None
329
+ self._lock = threading.Lock()
330
+ self._answers: dict = {}
331
+
332
+ # --- which model ---
333
+
334
+ def answer(self, task: str, kind: str = KIND_CHAT, lang: str = "") -> dict:
335
+ """The models route's whole answer for task: models (best first),
336
+ to_get, and for chat each model's window (windows); kept a minute."""
337
+ key = (task, kind, lang)
338
+ now = time.monotonic()
339
+ with self._lock:
340
+ kept = self._answers.get(key)
341
+ if kept and now - kept[0] < _MODELS_KEPT:
342
+ return kept[1]
343
+ q = {"task": task, "kind": kind}
344
+ if lang:
345
+ q["lang"] = lang
346
+ out = self.host.call("GET", "/host/v1/models?" + urllib.parse.urlencode(q)) or {}
347
+ with self._lock:
348
+ self._answers[key] = (now, out)
349
+ return out
350
+
351
+ def choices(self, task: str, kind: str = KIND_CHAT) -> List[str]:
352
+ """The models on this Mac that may do task for a call of kind, best
353
+ first. Empty: none may now (an optional task's feature stays hidden)."""
354
+ if self._test and self._test.get("model"):
355
+ return [self._test["model"]]
356
+ return [m for m in self.answer(task, kind).get("models") or [] if isinstance(m, str)]
357
+
358
+ def window(self, task: str, model: str = "") -> int:
359
+ """The window, in tokens for the prompt and the answer together, the
360
+ host runs model's chat calls in (the task's first model when none is
361
+ named); 0 when the host names none. Size every prompt from it."""
362
+ if not model:
363
+ ms = self.choices(task)
364
+ model = ms[0] if ms else ""
365
+ w = (self.answer(task).get("windows") or {}).get(model)
366
+ return w if isinstance(w, int) else 0
367
+
368
+ def model_for(self, task: str, kind: str = KIND_CHAT, want: str = "") -> str:
369
+ """The model to call for task: want, a person's pick, while the host
370
+ names it for the task; else the task's first."""
371
+ ms = self.choices(task, kind)
372
+ if not ms:
373
+ raise NoModel("nativecluster: no model on this Mac may do %s yet" % task)
374
+ return want if want in ms else ms[0]
375
+
376
+ def seats_for(self, model: str) -> int:
377
+ """How many of a model's calls may run at once across the group's
378
+ Macs: run about that many independent units, never more."""
379
+ state = self.host.call("GET", "/host/v1/line/state") or {}
380
+ n = (state.get("seats_for") or {}).get(model)
381
+ return n if isinstance(n, int) and n > 0 else 1
382
+
383
+ # --- a turn in line ---
384
+
385
+ def origin(self, origin: Optional[Origin] = None) -> Origin:
386
+ o = origin or current_origin()
387
+ if o is not None:
388
+ return o
389
+ t = (self._test or {}).get("ticket") or {}
390
+ if t.get("origin") == "request":
391
+ return Origin("request", person=t.get("person", ""), request=t.get("request", ""))
392
+ raise NoOrigin()
393
+
394
+ def wait(self, task: str, *, model: str = "", pick_chat: bool = False, origin: Optional[Origin] = None,
395
+ conversation: Optional[str] = None, waiting: Optional[Waiting] = None,
396
+ on_position: Optional[Callable[[int], None]] = None,
397
+ on_away: Optional[Callable[[str], None]] = None) -> Grant:
398
+ """Waits in the host's line for one call's seat, as origin. pick_chat
399
+ says the call is one chat call: the host may seat it on any of the
400
+ group's Macs holding the model, and names the model it seated it for.
401
+ on_away (else when_away's) hears the host's words when the wait is
402
+ held because the cluster is out of reach; the wait goes on."""
403
+ o = self.origin(origin)
404
+ if on_away is None:
405
+ on_away = current_away()
406
+ tag: dict = {"Person": o.person, "Task": task}
407
+ if model:
408
+ tag["Model"] = model
409
+ if o.job:
410
+ tag["Job"] = True
411
+ conv = current_conversation() if conversation is None else conversation
412
+ if conv:
413
+ tag["Conversation"] = conv
414
+ body: dict = {"kind": "acquire", "lane": o.lane, "tag": tag, "ticket": o.wire()}
415
+ if pick_chat:
416
+ body["pick"] = "chat"
417
+ conn, resp = self.host.open("POST", "/host/v1/line/wait", json.dumps(body).encode("utf-8"),
418
+ {"Content-Type": "application/json"}, timeout=None,
419
+ sent=waiting._attach if waiting is not None else None)
420
+ try:
421
+ if resp.status != 200:
422
+ raise HostError("POST", "/host/v1/line/wait", resp.status, resp.read(512).decode("utf-8", "replace"))
423
+ while True:
424
+ line = resp.readline(1 << 20)
425
+ if not line and waiting is not None and waiting.cancelled:
426
+ raise Refused("nativecluster: the wait was given up")
427
+ if not line:
428
+ raise HostError("POST", "/host/v1/line/wait", 200, "the line ended the wait without an answer")
429
+ if not line.strip():
430
+ continue
431
+ msg = json.loads(line)
432
+ if msg.get("refused"):
433
+ raise Refused("nativecluster: the host's line refused the call: %s" % msg["refused"], msg["refused"])
434
+ if msg.get("id"):
435
+ g = Grant(id=msg["id"], cache_key=msg.get("cache_key") or "", model=msg.get("model") or "",
436
+ window=_int(msg.get("window")))
437
+ if waiting is not None and not waiting._deliver(g):
438
+ self.release(g, model)
439
+ raise Refused("nativecluster: the wait was given up")
440
+ return g
441
+ if on_position is not None and _int(msg.get("position")) > 0:
442
+ on_position(msg["position"])
443
+ if on_away is not None and isinstance(msg.get("away"), str) and msg["away"]:
444
+ on_away(msg["away"])
445
+ except OSError:
446
+ if waiting is not None and waiting.cancelled:
447
+ raise Refused("nativecluster: the wait was given up") from None
448
+ raise
449
+ finally:
450
+ close(conn, resp)
451
+
452
+ def release(self, grant: Grant, model: str = "", read: int = 0, written: int = 0) -> None:
453
+ """Gives a grant back with what its call did: counts, never words."""
454
+ try:
455
+ self.host.call("POST", "/host/v1/line/release", {
456
+ "grant": grant.id,
457
+ "notes": {"model": model or grant.model, "calls": 1, "read": read, "written": written},
458
+ })
459
+ except (OSError, HostError):
460
+ pass # the host ends the grant with the app's process at the latest
461
+
462
+ def give_back(self, grant: Grant) -> None:
463
+ """Gives back a grant no call used (a wait given up as it was
464
+ answered): nothing to count."""
465
+ try:
466
+ self.host.call("POST", "/host/v1/line/release", {"grant": grant.id})
467
+ except (OSError, HostError):
468
+ pass # the host ends the grant with the app's process at the latest
469
+
470
+ def post(self, grant: Grant, path: str, content_type: str, body: bytes):
471
+ """Sends one model call on grant: its connection and unread answer."""
472
+ return self.host.open("POST", path, body, {GRANT_HEADER: grant.id, "Content-Type": content_type}, timeout=None)
473
+
474
+ # --- calls ---
475
+
476
+ def _chat_body(self, model: str, grant: Grant, messages: list, max_tokens: int, extra: dict, stream: bool) -> bytes:
477
+ b = dict(extra)
478
+ b["model"] = model
479
+ b["messages"] = messages
480
+ b["stream"] = stream
481
+ if max_tokens > 0:
482
+ b["max_tokens"] = max_tokens
483
+ if stream:
484
+ b["stream_options"] = {"include_usage": True}
485
+ if grant.cache_key:
486
+ b["cache_key"] = grant.cache_key
487
+ return json.dumps(b).encode("utf-8")
488
+
489
+ def chat(self, task: str, messages: list, *, max_tokens: int = 0, origin: Optional[Origin] = None, **extra: Any) -> Answer:
490
+ """A chat model's whole answer for task. extra carries any other
491
+ OpenAI-shaped field (response_format, temperature, tools), never
492
+ model or stream."""
493
+ o = self.origin(origin)
494
+ asked = self.model_for(task)
495
+ for attempt in range(MAX_TRIES):
496
+ g = self.wait(task, model=asked, pick_chat=True, origin=o)
497
+ model = g.model or asked
498
+ conn, resp = self.post(g, "/v1/chat/completions", "application/json",
499
+ self._chat_body(model, g, messages, max_tokens, extra, False))
500
+ try:
501
+ raw = resp.read(16 << 20)
502
+ except BaseException:
503
+ self.release(g, model)
504
+ raise
505
+ finally:
506
+ close(conn, resp)
507
+ if stopped(resp.headers):
508
+ self.release(g, model)
509
+ if attempt + 1 < MAX_TRIES:
510
+ continue
511
+ raise Stopped("nativecluster: the waiting line stopped this answer to make way for someone; try again")
512
+ read, written = usage_of(raw)
513
+ self.release(g, model, read, written)
514
+ if resp.status != 200:
515
+ raise _model_error(resp.status, raw)
516
+ try:
517
+ text = json.loads(raw)["choices"][0]["message"].get("content") or ""
518
+ except (ValueError, KeyError, IndexError, TypeError, AttributeError):
519
+ raise ModelError(resp.status, "the model's answer did not read") from None
520
+ return Answer(text=text, model=model, read=read, written=written)
521
+ raise AssertionError("unreachable")
522
+
523
+ def chat_stream(self, task: str, messages: list, on_delta: Optional[Callable[[str], None]] = None, *,
524
+ max_tokens: int = 0, origin: Optional[Origin] = None, **extra: Any) -> Answer:
525
+ """A chat model's answer for task, each piece passed to on_delta as it
526
+ is written. A stop before the first piece runs the call again; after
527
+ it, the answer ends with Stopped."""
528
+ o = self.origin(origin)
529
+ asked = self.model_for(task)
530
+ for attempt in range(MAX_TRIES):
531
+ g = self.wait(task, model=asked, pick_chat=True, origin=o)
532
+ model = g.model or asked
533
+ scan = EventScan()
534
+ text: List[str] = []
535
+ try:
536
+ self._stream(g, model, messages, max_tokens, extra, scan, text, on_delta)
537
+ except Stopped:
538
+ if not scan.began and attempt + 1 < MAX_TRIES:
539
+ continue
540
+ raise
541
+ finally:
542
+ self.release(g, model, scan.read, scan.written)
543
+ return Answer(text="".join(text), model=model, read=scan.read, written=scan.written)
544
+ raise AssertionError("unreachable")
545
+
546
+ def _stream(self, g: Grant, model: str, messages: list, max_tokens: int, extra: dict, scan: EventScan,
547
+ text: List[str], on_delta: Optional[Callable[[str], None]]) -> None:
548
+ conn, resp = self.post(g, "/v1/chat/completions", "application/json",
549
+ self._chat_body(model, g, messages, max_tokens, extra, True))
550
+ try:
551
+ if stopped(resp.headers):
552
+ raise Stopped("nativecluster: the waiting line stopped this answer to make way for someone; try again")
553
+ if resp.status != 200:
554
+ raise _model_error(resp.status, resp.read(64 << 10))
555
+ while not scan.done:
556
+ chunk = resp.read1(64 << 10) if hasattr(resp, "read1") else resp.read(4096)
557
+ if not chunk:
558
+ break
559
+ _, pieces = scan.feed(chunk)
560
+ self._pieces(scan, pieces, text, on_delta)
561
+ if not scan.done:
562
+ _, pieces = scan.flush()
563
+ self._pieces(scan, pieces, text, on_delta)
564
+ if scan.error is not None:
565
+ raise _model_error(200, json.dumps({"error": scan.error}).encode("utf-8"))
566
+ if not scan.done:
567
+ # The host says why an answer ended in a trailer this client
568
+ # cannot read: the line stopping it is the usual reason.
569
+ raise Stopped("nativecluster: the answer ended early (the waiting line may have stopped it to make way for someone); try again")
570
+ finally:
571
+ close(conn, resp)
572
+
573
+ @staticmethod
574
+ def _pieces(scan: EventScan, pieces: List[str], text: List[str], on_delta: Optional[Callable[[str], None]]) -> None:
575
+ for p in pieces:
576
+ text.append(p)
577
+ if on_delta is not None:
578
+ on_delta(p)
579
+
580
+ def embed(self, task: str, inputs: List[str], *, origin: Optional[Origin] = None) -> List[List[float]]:
581
+ """A vector for each input, from an embedding model for task."""
582
+ o = self.origin(origin)
583
+ model = self.model_for(task, KIND_EMBEDDING)
584
+ raw = self._once(task, model, o, "/v1/embeddings", "application/json",
585
+ json.dumps({"model": model, "input": inputs}).encode("utf-8"))
586
+ vecs: List[List[float]] = [[] for _ in inputs]
587
+ for d in json.loads(raw).get("data") or []:
588
+ i = d.get("index")
589
+ if isinstance(i, int) and 0 <= i < len(vecs):
590
+ vecs[i] = d.get("embedding") or []
591
+ return vecs
592
+
593
+ def transcribe(self, task: str, audio: bytes, filename: str, *, origin: Optional[Origin] = None) -> str:
594
+ """The text of a recording (a file's bytes, named so its kind shows),
595
+ from a speech model for task."""
596
+ o = self.origin(origin)
597
+ model = self.model_for(task, KIND_TRANSCRIPTION)
598
+ boundary = "nativecluster" + "%x" % int(time.time() * 1e6)
599
+ body = (
600
+ ("--%s\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n%s\r\n" % (boundary, model)).encode("utf-8")
601
+ + ("--%s\r\nContent-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\n"
602
+ "Content-Type: application/octet-stream\r\n\r\n" % (boundary, filename.replace('"', ""))).encode("utf-8")
603
+ + audio + ("\r\n--%s--\r\n" % boundary).encode("utf-8")
604
+ )
605
+ raw = self._once(task, model, o, "/v1/audio/transcriptions", "multipart/form-data; boundary=" + boundary, body)
606
+ return json.loads(raw).get("text") or ""
607
+
608
+ def _once(self, task: str, model: str, origin: Origin, path: str, content_type: str, body: bytes) -> bytes:
609
+ """One call that is not streamed, run again if the line stops it."""
610
+ for attempt in range(MAX_TRIES):
611
+ g = self.wait(task, model=model, origin=origin)
612
+ try:
613
+ conn, resp = self.post(g, path, content_type, body)
614
+ try:
615
+ raw = resp.read(64 << 20)
616
+ finally:
617
+ close(conn, resp)
618
+ finally:
619
+ self.release(g, model)
620
+ if stopped(resp.headers):
621
+ if attempt + 1 < MAX_TRIES:
622
+ continue
623
+ raise Stopped("nativecluster: the waiting line stopped this call to make way for someone; try again")
624
+ if resp.status != 200:
625
+ raise _model_error(resp.status, raw)
626
+ return raw
627
+ raise AssertionError("unreachable")
628
+
629
+
630
+ def _model_error(status: int, raw: bytes) -> ModelError:
631
+ try:
632
+ err = json.loads(raw).get("error", raw.decode("utf-8", "replace"))
633
+ except (ValueError, AttributeError):
634
+ err = raw.decode("utf-8", "replace")
635
+ if overflow_error(err) is not None:
636
+ return ContextTooLong(status, overflow_error(err))
637
+ if isinstance(err, dict) and err.get("code") == CONTEXT_LENGTH_EXCEEDED:
638
+ return ContextTooLong(status, err)
639
+ return ModelError(status, err)