nativecluster 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nativecluster/__init__.py +139 -0
- nativecluster/_firings.py +185 -0
- nativecluster/_gate.py +312 -0
- nativecluster/_host.py +337 -0
- nativecluster/_hostsock.py +105 -0
- nativecluster/_launch.py +203 -0
- nativecluster/_models.py +639 -0
- nativecluster/_origin.py +217 -0
- nativecluster/_serve.py +227 -0
- nativecluster/_ticket.py +209 -0
- nativecluster/transport.py +576 -0
- nativecluster-0.1.0.dist-info/METADATA +107 -0
- nativecluster-0.1.0.dist-info/RECORD +16 -0
- nativecluster-0.1.0.dist-info/WHEEL +5 -0
- nativecluster-0.1.0.dist-info/licenses/LICENSE +202 -0
- nativecluster-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,576 @@
|
|
|
1
|
+
"""OpenAI's Python library, through the host.
|
|
2
|
+
|
|
3
|
+
An app built on OpenAI's library (an agent ported to NativeCluster) keeps
|
|
4
|
+
it: it hands the library this module's HTTP client, and every model call the
|
|
5
|
+
library makes waits its turn in the host's one line as whatever started it
|
|
6
|
+
(the request being handled, a firing, a job: the kit's origin), runs on the
|
|
7
|
+
grant the line gave it, and gives the grant back when its answer ends, read
|
|
8
|
+
to the end or not. A call's "model" names one of the app's tasks, never a
|
|
9
|
+
model: the kit asks the host which of the group's models may do it, and the
|
|
10
|
+
host sends the call to that one. The app never reaches the engine. An app
|
|
11
|
+
that lets a person pick among them (a model menu) names the pick after the
|
|
12
|
+
task, "<task>/<model>", one of the models the host names for that task
|
|
13
|
+
(Models.choices); a model it no longer names gives way to the task's
|
|
14
|
+
first, so a pick never breaks a call.
|
|
15
|
+
|
|
16
|
+
client = OpenAI(base_url=transport.BASE_URL, api_key="nativecluster",
|
|
17
|
+
http_client=transport.http_client())
|
|
18
|
+
client.chat.completions.create(model="<one of the app's tasks>", messages=[...])
|
|
19
|
+
|
|
20
|
+
What the line says comes back in OpenAI's shapes, so the library's own
|
|
21
|
+
handling applies: a call the line stopped to make way for someone is run
|
|
22
|
+
again (a stream already under way ends with an error frame instead); a
|
|
23
|
+
prompt longer than the model's window is OpenAI's context_length_exceeded;
|
|
24
|
+
no origin is a 403, no model for the task a 404 (model_not_found), a refused
|
|
25
|
+
wait a 503. It needs httpx, which OpenAI's library carries.
|
|
26
|
+
|
|
27
|
+
An app that stops a call from another thread (its person closed the window
|
|
28
|
+
while the agent's thread waits in line or reads the answer) gives the call
|
|
29
|
+
up with cancel(client): the wait leaves the line, the grant goes back, the
|
|
30
|
+
model stops writing wherever the host placed the call, and the thread
|
|
31
|
+
making the call gets Cancelled.
|
|
32
|
+
|
|
33
|
+
transport.cancel(client) # the OpenAI client, or http_client()'s
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import asyncio
|
|
39
|
+
import json
|
|
40
|
+
import threading
|
|
41
|
+
import weakref
|
|
42
|
+
from typing import Any, AsyncIterator, Callable, Iterator, Optional, Union
|
|
43
|
+
|
|
44
|
+
import httpx
|
|
45
|
+
|
|
46
|
+
from ._hostsock import BASE_URL as _HOST_URL
|
|
47
|
+
from ._hostsock import HostError, HostSocket
|
|
48
|
+
from ._launch import Launch, launch
|
|
49
|
+
from ._models import (
|
|
50
|
+
GRANT_HEADER,
|
|
51
|
+
KIND_CHAT,
|
|
52
|
+
KIND_TRANSCRIPTION,
|
|
53
|
+
MAX_TRIES,
|
|
54
|
+
ROUTES,
|
|
55
|
+
EventScan,
|
|
56
|
+
Grant,
|
|
57
|
+
Models,
|
|
58
|
+
NoModel,
|
|
59
|
+
NoOrigin,
|
|
60
|
+
Refused,
|
|
61
|
+
Waiting,
|
|
62
|
+
multipart_field,
|
|
63
|
+
overflow_body,
|
|
64
|
+
stopped,
|
|
65
|
+
usage_of,
|
|
66
|
+
)
|
|
67
|
+
from ._origin import Origin
|
|
68
|
+
|
|
69
|
+
# BASE_URL is the base URL to give OpenAI's library: the host's socket is the
|
|
70
|
+
# destination, whatever the URL says.
|
|
71
|
+
BASE_URL = _HOST_URL + "/v1"
|
|
72
|
+
|
|
73
|
+
# TIMEOUT waits as long as an answer takes to read (a long answer streams
|
|
74
|
+
# for minutes), and bounds the rest. Waiting in line is not counted in it.
|
|
75
|
+
TIMEOUT = httpx.Timeout(connect=15.0, read=None, write=15.0, pool=10.0)
|
|
76
|
+
|
|
77
|
+
_EARLY = (
|
|
78
|
+
b'data: {"error": {"message": "the answer ended early: the waiting line may have stopped it to make way for '
|
|
79
|
+
b'someone; try again", "type": "server_error", "code": "nc_stopped"}}\n\n'
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class Cancelled(httpx.TransportError):
|
|
84
|
+
"""A model call cancel() gave up: it left the line, or its grant went
|
|
85
|
+
back and its answer ended. OpenAI's library reads it as a connection
|
|
86
|
+
error (APIConnectionError), and every retry of it on the same client is
|
|
87
|
+
refused at once, so nothing waits in line again."""
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _cancelled(request: Optional[httpx.Request] = None) -> Cancelled:
|
|
91
|
+
return Cancelled("nativecluster: this model call was given up (transport.cancel)", request=request)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _later(fn: Callable[..., None], *args: Any) -> None:
|
|
95
|
+
"""fn(*args) on a thread of its own: a give-back from cancel never
|
|
96
|
+
waits for the host, and happens even while the caller unwinds."""
|
|
97
|
+
threading.Thread(target=fn, args=args, daemon=True).start()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class _Held:
|
|
101
|
+
"""The grant one call holds, given back once: by the call as its
|
|
102
|
+
answer ends (with what it did), or by cancel."""
|
|
103
|
+
|
|
104
|
+
def __init__(self, models: Models, grant: Grant, model: str) -> None:
|
|
105
|
+
self._models = models
|
|
106
|
+
self.grant = grant
|
|
107
|
+
self.model = model
|
|
108
|
+
self._lock = threading.Lock()
|
|
109
|
+
self._given = False
|
|
110
|
+
|
|
111
|
+
def give_back(self, read: int = 0, written: int = 0) -> None:
|
|
112
|
+
with self._lock:
|
|
113
|
+
if self._given:
|
|
114
|
+
return
|
|
115
|
+
self._given = True
|
|
116
|
+
self._models.release(self.grant, self.model, read, written)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _error(status: int, message: str, code: str, request: httpx.Request, retry_after: int = 0) -> httpx.Response:
|
|
120
|
+
headers = {"Retry-After": str(retry_after)} if retry_after else {}
|
|
121
|
+
body = {"error": {"message": message, "type": "invalid_request_error" if status < 500 else "server_error", "code": code}}
|
|
122
|
+
return httpx.Response(status, json=body, headers=headers, request=request)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class _Call:
|
|
126
|
+
"""One model call's plan: its kind, the task its "model" names, and its
|
|
127
|
+
body made for the model and the turn the host gave it."""
|
|
128
|
+
|
|
129
|
+
def __init__(self, kind: str, model: str, body_for: Callable[[str, Grant], bytes]) -> None:
|
|
130
|
+
self.kind = kind
|
|
131
|
+
# "<task>" or "<task>/<model>": a task's id holds no "/", nor does a
|
|
132
|
+
# catalog model's name.
|
|
133
|
+
self.task, _, self.want = model.partition("/")
|
|
134
|
+
self.body_for = body_for
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _plan(request: httpx.Request, body: bytes) -> Union[_Call, httpx.Response, None]:
|
|
138
|
+
"""request's plan, an error to answer at once, or None for a request
|
|
139
|
+
that is no model call (it goes to the host as it is)."""
|
|
140
|
+
kind = ROUTES.get(request.url.path.rstrip("/"))
|
|
141
|
+
if kind is None or request.method != "POST":
|
|
142
|
+
return None
|
|
143
|
+
if kind == KIND_TRANSCRIPTION:
|
|
144
|
+
ctype = request.headers.get("content-type", "")
|
|
145
|
+
_, task = multipart_field(body, ctype, "model")
|
|
146
|
+
if not task:
|
|
147
|
+
return _error(400, "a model call names one of the app's tasks as its model", "nc_no_task", request)
|
|
148
|
+
return _Call(kind, task, lambda model, g: multipart_field(body, ctype, "model", model)[0])
|
|
149
|
+
try:
|
|
150
|
+
obj = json.loads(body)
|
|
151
|
+
except ValueError:
|
|
152
|
+
obj = None
|
|
153
|
+
if not isinstance(obj, dict) or not isinstance(obj.get("model"), str) or not obj["model"]:
|
|
154
|
+
return _error(400, "a model call names one of the app's tasks as its model", "nc_no_task", request)
|
|
155
|
+
|
|
156
|
+
def body_for(model: str, g: Grant) -> bytes:
|
|
157
|
+
b = dict(obj)
|
|
158
|
+
b["model"] = model
|
|
159
|
+
if kind == KIND_CHAT and g.cache_key:
|
|
160
|
+
b["cache_key"] = g.cache_key
|
|
161
|
+
return json.dumps(b).encode("utf-8")
|
|
162
|
+
|
|
163
|
+
return _Call(kind, obj["model"], body_for)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _to_host(request: httpx.Request, credential: str, content: Optional[bytes] = None, grant: Optional[Grant] = None) -> httpx.Request:
|
|
167
|
+
"""request, sent to the host with the app's credential (whatever key the
|
|
168
|
+
library sent) and the grant it runs on."""
|
|
169
|
+
headers = [(k, v) for k, v in request.headers.multi_items()
|
|
170
|
+
if k.lower() not in ("authorization", "content-length", "transfer-encoding", "host", GRANT_HEADER.lower())]
|
|
171
|
+
headers.append(("Authorization", "Bearer " + credential))
|
|
172
|
+
if grant is not None:
|
|
173
|
+
headers.append((GRANT_HEADER, grant.id))
|
|
174
|
+
return httpx.Request(request.method, request.url, headers=headers,
|
|
175
|
+
content=request.content if content is None else content, extensions=request.extensions)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _streams(resp: httpx.Response) -> bool:
|
|
179
|
+
return resp.status_code == 200 and resp.headers.get("content-type", "").startswith("text/event-stream")
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _whole(resp: httpx.Response, raw: bytes, request: httpx.Request) -> httpx.Response:
|
|
183
|
+
headers = [(k, v) for k, v in resp.headers.multi_items() if k.lower() not in ("content-length", "transfer-encoding")]
|
|
184
|
+
return httpx.Response(resp.status_code, headers=headers, content=overflow_body(resp.status_code, raw),
|
|
185
|
+
request=request, extensions=resp.extensions)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _refusal(e: Exception, request: httpx.Request) -> httpx.Response:
|
|
189
|
+
if isinstance(e, NoOrigin):
|
|
190
|
+
return _error(403, str(e), "nc_no_origin", request)
|
|
191
|
+
if isinstance(e, NoModel):
|
|
192
|
+
return _error(404, str(e), "model_not_found", request)
|
|
193
|
+
if isinstance(e, HostError):
|
|
194
|
+
words = _host_words(e)
|
|
195
|
+
if words:
|
|
196
|
+
# NativeCluster's own words to the person (this computer's switch
|
|
197
|
+
# is off, it is not added to the cluster or was removed, Members
|
|
198
|
+
# does not give them the app): a refusal to show as it is.
|
|
199
|
+
return _error(503, words, "nc_refused", request)
|
|
200
|
+
return _error(e.status if e.status >= 400 else 502, str(e), "nc_host", request)
|
|
201
|
+
if isinstance(e, Refused) and e.words:
|
|
202
|
+
# The line's own words for the person, as they are.
|
|
203
|
+
return _error(503, e.words, "nc_refused", request)
|
|
204
|
+
return _error(503, str(e), "nc_refused", request)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
# The host's codes for an answer whose error is words for the person.
|
|
208
|
+
_PERSON_CODES = ("inference_off", "computer_waiting", "computer_removed", "not_given", "app_not_here")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _host_words(e: "HostError") -> str:
|
|
212
|
+
try:
|
|
213
|
+
body = json.loads(e.text)
|
|
214
|
+
except ValueError:
|
|
215
|
+
return ""
|
|
216
|
+
if isinstance(body, dict) and body.get("code") in _PERSON_CODES and isinstance(body.get("error"), str):
|
|
217
|
+
return body["error"]
|
|
218
|
+
return ""
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
class _GrantStream(httpx.SyncByteStream):
|
|
222
|
+
"""A streamed answer passed on as it comes, its grant given back once it
|
|
223
|
+
ends: read to the end, or closed early. cancelled says whether cancel()
|
|
224
|
+
gave the call up: its answer, ended by the host, raises Cancelled rather
|
|
225
|
+
than reading as a stream the line stopped."""
|
|
226
|
+
|
|
227
|
+
def __init__(self, resp: httpx.Response, give_back: Callable[[int, int], None],
|
|
228
|
+
cancelled: Optional[Callable[[], bool]] = None) -> None:
|
|
229
|
+
self._resp = resp
|
|
230
|
+
self._give_back = give_back
|
|
231
|
+
self._cancelled = cancelled or (lambda: False)
|
|
232
|
+
self._scan = EventScan()
|
|
233
|
+
self._lock = threading.Lock()
|
|
234
|
+
self._given = False
|
|
235
|
+
|
|
236
|
+
def _end(self) -> None:
|
|
237
|
+
with self._lock:
|
|
238
|
+
if self._given:
|
|
239
|
+
return
|
|
240
|
+
self._given = True
|
|
241
|
+
self._give_back(self._scan.read, self._scan.written)
|
|
242
|
+
|
|
243
|
+
def __iter__(self) -> Iterator[bytes]:
|
|
244
|
+
try:
|
|
245
|
+
for chunk in self._resp.iter_raw():
|
|
246
|
+
out, _ = self._scan.feed(chunk)
|
|
247
|
+
if out:
|
|
248
|
+
yield out
|
|
249
|
+
except Exception as e:
|
|
250
|
+
if self._cancelled():
|
|
251
|
+
self._end()
|
|
252
|
+
raise _cancelled() from e
|
|
253
|
+
raise
|
|
254
|
+
out, _ = self._scan.flush()
|
|
255
|
+
if out:
|
|
256
|
+
yield out
|
|
257
|
+
self._end()
|
|
258
|
+
if not self._scan.done and self._scan.error is None:
|
|
259
|
+
if self._cancelled():
|
|
260
|
+
raise _cancelled()
|
|
261
|
+
yield _EARLY
|
|
262
|
+
|
|
263
|
+
def close(self) -> None:
|
|
264
|
+
try:
|
|
265
|
+
self._resp.close()
|
|
266
|
+
finally:
|
|
267
|
+
self._end()
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
class _AsyncGrantStream(httpx.AsyncByteStream):
|
|
271
|
+
def __init__(self, resp: httpx.Response, give_back: Callable[[int, int], None]) -> None:
|
|
272
|
+
self._resp = resp
|
|
273
|
+
self._give_back = give_back
|
|
274
|
+
self._scan = EventScan()
|
|
275
|
+
self._lock = threading.Lock()
|
|
276
|
+
self._given = False
|
|
277
|
+
|
|
278
|
+
def _end(self) -> None:
|
|
279
|
+
with self._lock:
|
|
280
|
+
if self._given:
|
|
281
|
+
return
|
|
282
|
+
self._given = True
|
|
283
|
+
self._give_back(self._scan.read, self._scan.written)
|
|
284
|
+
|
|
285
|
+
async def __aiter__(self) -> AsyncIterator[bytes]:
|
|
286
|
+
async for chunk in self._resp.aiter_raw():
|
|
287
|
+
out, _ = self._scan.feed(chunk)
|
|
288
|
+
if out:
|
|
289
|
+
yield out
|
|
290
|
+
out, _ = self._scan.flush()
|
|
291
|
+
if out:
|
|
292
|
+
yield out
|
|
293
|
+
self._end()
|
|
294
|
+
if not self._scan.done and self._scan.error is None:
|
|
295
|
+
yield _EARLY
|
|
296
|
+
|
|
297
|
+
async def aclose(self) -> None:
|
|
298
|
+
try:
|
|
299
|
+
await self._resp.aclose()
|
|
300
|
+
finally:
|
|
301
|
+
self._end()
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _models_for(launched: Optional[Launch], models: Optional[Models]) -> "tuple[Launch, Models]":
|
|
305
|
+
l = launched or launch()
|
|
306
|
+
return l, models or Models(HostSocket(l.host_socket, l.credential), test=l.test)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
class HostTransport(httpx.BaseTransport):
|
|
310
|
+
"""httpx's transport for OpenAI's library behind the host (sync)."""
|
|
311
|
+
|
|
312
|
+
def __init__(self, launched: Optional[Launch] = None, *, models: Optional[Models] = None) -> None:
|
|
313
|
+
l, self._models = _models_for(launched, models)
|
|
314
|
+
self._credential = l.credential
|
|
315
|
+
self._inner = httpx.HTTPTransport(uds=l.host_socket)
|
|
316
|
+
# What cancel gives up: the waits in line now and the grants calls
|
|
317
|
+
# hold now.
|
|
318
|
+
self._lock = threading.Lock()
|
|
319
|
+
self._cancelled = False
|
|
320
|
+
self._waits: "set[Waiting]" = set()
|
|
321
|
+
self._held: "set[_Held]" = set()
|
|
322
|
+
|
|
323
|
+
def cancel(self) -> None:
|
|
324
|
+
"""Gives up every model call this transport is making, and every
|
|
325
|
+
one made on it after: a cancelled transport is done, so a retry
|
|
326
|
+
never waits in line again (make a new client for new calls). Safe
|
|
327
|
+
from any thread, and returns at once, without waiting for the host:
|
|
328
|
+
|
|
329
|
+
- a call still waiting in line leaves it: its held wait's
|
|
330
|
+
connection is shut down (never closed under its thread), and the
|
|
331
|
+
host gives its place away;
|
|
332
|
+
- a call that holds a grant gives it back (POST
|
|
333
|
+
/host/v1/line/release, on a thread of its own): the host stops
|
|
334
|
+
the call at its engine, on this Mac or wherever it placed it, and
|
|
335
|
+
ends its answer;
|
|
336
|
+
- the thread making the call gets Cancelled, whether it waited in
|
|
337
|
+
line, waited for the answer or read the stream.
|
|
338
|
+
|
|
339
|
+
Calls that are no model call (the models list) are not touched."""
|
|
340
|
+
with self._lock:
|
|
341
|
+
self._cancelled = True
|
|
342
|
+
waits, self._waits = self._waits, set()
|
|
343
|
+
held, self._held = self._held, set()
|
|
344
|
+
# Under the lock, so a wait answered meanwhile finds itself
|
|
345
|
+
# given up and leaves its grant to this (_wait).
|
|
346
|
+
unused = [w.cancel() for w in waits]
|
|
347
|
+
for g in unused:
|
|
348
|
+
if g is not None:
|
|
349
|
+
_later(self._models.give_back, g)
|
|
350
|
+
for h in held:
|
|
351
|
+
_later(h.give_back)
|
|
352
|
+
|
|
353
|
+
@property
|
|
354
|
+
def cancelled(self) -> bool:
|
|
355
|
+
"""Whether cancel() gave this transport's calls up."""
|
|
356
|
+
return self._cancelled
|
|
357
|
+
|
|
358
|
+
def _wait(self, call: _Call, model: str, origin: Origin, request: httpx.Request) -> Union[_Held, httpx.Response]:
|
|
359
|
+
"""One wait in line for call, which cancel can give up: the grant it
|
|
360
|
+
got, held, or the line's refusal to answer with."""
|
|
361
|
+
waiting = Waiting()
|
|
362
|
+
with self._lock:
|
|
363
|
+
if self._cancelled:
|
|
364
|
+
raise _cancelled(request)
|
|
365
|
+
self._waits.add(waiting)
|
|
366
|
+
try:
|
|
367
|
+
g = self._models.wait(call.task, model=model, pick_chat=call.kind == KIND_CHAT, origin=origin,
|
|
368
|
+
waiting=waiting)
|
|
369
|
+
except (Refused, HostError) as e:
|
|
370
|
+
with self._lock:
|
|
371
|
+
self._waits.discard(waiting)
|
|
372
|
+
if self._cancelled:
|
|
373
|
+
raise _cancelled(request) from e
|
|
374
|
+
return _refusal(e, request)
|
|
375
|
+
except OSError as e:
|
|
376
|
+
with self._lock:
|
|
377
|
+
self._waits.discard(waiting)
|
|
378
|
+
if self._cancelled:
|
|
379
|
+
raise _cancelled(request) from e
|
|
380
|
+
raise httpx.ConnectError(str(e), request=request) from e
|
|
381
|
+
held = _Held(self._models, g, g.model or model)
|
|
382
|
+
with self._lock:
|
|
383
|
+
self._waits.discard(waiting)
|
|
384
|
+
given_up = waiting.cancelled
|
|
385
|
+
if not given_up:
|
|
386
|
+
waiting.take()
|
|
387
|
+
self._held.add(held)
|
|
388
|
+
if given_up:
|
|
389
|
+
# cancel reached the wait as it was answered: cancel gives the
|
|
390
|
+
# grant back.
|
|
391
|
+
raise _cancelled(request)
|
|
392
|
+
return held
|
|
393
|
+
|
|
394
|
+
def _give_back(self, held: _Held, read: int = 0, written: int = 0) -> None:
|
|
395
|
+
with self._lock:
|
|
396
|
+
self._held.discard(held)
|
|
397
|
+
held.give_back(read, written)
|
|
398
|
+
|
|
399
|
+
def handle_request(self, request: httpx.Request) -> httpx.Response:
|
|
400
|
+
call = _plan(request, request.read())
|
|
401
|
+
if call is None:
|
|
402
|
+
return self._inner.handle_request(_to_host(request, self._credential))
|
|
403
|
+
if isinstance(call, httpx.Response):
|
|
404
|
+
return call
|
|
405
|
+
try:
|
|
406
|
+
origin = self._models.origin()
|
|
407
|
+
model = self._models.model_for(call.task, call.kind, call.want)
|
|
408
|
+
except (NoOrigin, NoModel, HostError) as e:
|
|
409
|
+
return _refusal(e, request)
|
|
410
|
+
except OSError as e:
|
|
411
|
+
raise httpx.ConnectError(str(e), request=request) from e
|
|
412
|
+
for _ in range(MAX_TRIES):
|
|
413
|
+
held = self._wait(call, model, origin, request)
|
|
414
|
+
if isinstance(held, httpx.Response):
|
|
415
|
+
return held
|
|
416
|
+
g, used = held.grant, held.model
|
|
417
|
+
try:
|
|
418
|
+
resp = self._inner.handle_request(_to_host(request, self._credential, call.body_for(used, g), g))
|
|
419
|
+
except BaseException as e:
|
|
420
|
+
self._give_back(held)
|
|
421
|
+
if self._cancelled:
|
|
422
|
+
raise _cancelled(request) from e
|
|
423
|
+
raise
|
|
424
|
+
if stopped(resp.headers):
|
|
425
|
+
resp.close()
|
|
426
|
+
self._give_back(held)
|
|
427
|
+
continue
|
|
428
|
+
if _streams(resp):
|
|
429
|
+
return httpx.Response(resp.status_code, headers=resp.headers, request=request, extensions=resp.extensions,
|
|
430
|
+
stream=_GrantStream(resp, lambda r, w, held=held: self._give_back(held, r, w),
|
|
431
|
+
cancelled=lambda: self._cancelled))
|
|
432
|
+
raw = b""
|
|
433
|
+
try:
|
|
434
|
+
for chunk in resp.iter_raw():
|
|
435
|
+
raw += chunk
|
|
436
|
+
except Exception as e:
|
|
437
|
+
if self._cancelled:
|
|
438
|
+
raise _cancelled(request) from e
|
|
439
|
+
raise
|
|
440
|
+
finally:
|
|
441
|
+
resp.close()
|
|
442
|
+
self._give_back(held, *usage_of(raw))
|
|
443
|
+
if self._cancelled:
|
|
444
|
+
raise _cancelled(request)
|
|
445
|
+
return _whole(resp, raw, request)
|
|
446
|
+
return _error(503, "the waiting line stopped this call to make way for someone; try again", "nc_stopped",
|
|
447
|
+
request, retry_after=1)
|
|
448
|
+
|
|
449
|
+
def close(self) -> None:
|
|
450
|
+
self._inner.close()
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
class AsyncHostTransport(httpx.AsyncBaseTransport):
|
|
454
|
+
"""httpx's transport for OpenAI's library behind the host (asyncio).
|
|
455
|
+
The kit's waits in line run on threads, so the event loop never waits
|
|
456
|
+
for one; a call cancelled while it waits gives its place in line up."""
|
|
457
|
+
|
|
458
|
+
def __init__(self, launched: Optional[Launch] = None, *, models: Optional[Models] = None) -> None:
|
|
459
|
+
l, self._models = _models_for(launched, models)
|
|
460
|
+
self._credential = l.credential
|
|
461
|
+
self._inner = httpx.AsyncHTTPTransport(uds=l.host_socket)
|
|
462
|
+
|
|
463
|
+
def _release_later(self, g: Grant, model: str, read: int = 0, written: int = 0) -> None:
|
|
464
|
+
# On a thread of its own: a release must happen even while the
|
|
465
|
+
# caller is being cancelled.
|
|
466
|
+
threading.Thread(target=self._models.release, args=(g, model, read, written), daemon=True).start()
|
|
467
|
+
|
|
468
|
+
async def _wait(self, call: _Call, model: str, origin: Origin) -> Grant:
|
|
469
|
+
waiting = Waiting()
|
|
470
|
+
try:
|
|
471
|
+
await asyncio.to_thread(self._models.wait, call.task, model=model, pick_chat=call.kind == KIND_CHAT,
|
|
472
|
+
origin=origin, waiting=waiting)
|
|
473
|
+
except asyncio.CancelledError:
|
|
474
|
+
left = waiting.cancel()
|
|
475
|
+
if left is not None:
|
|
476
|
+
self._release_later(left, left.model or model)
|
|
477
|
+
raise
|
|
478
|
+
g = waiting.take()
|
|
479
|
+
assert g is not None
|
|
480
|
+
return g
|
|
481
|
+
|
|
482
|
+
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
|
483
|
+
call = _plan(request, await request.aread())
|
|
484
|
+
if call is None:
|
|
485
|
+
return await self._inner.handle_async_request(_to_host(request, self._credential))
|
|
486
|
+
if isinstance(call, httpx.Response):
|
|
487
|
+
return call
|
|
488
|
+
try:
|
|
489
|
+
origin = self._models.origin()
|
|
490
|
+
model = await asyncio.to_thread(self._models.model_for, call.task, call.kind, call.want)
|
|
491
|
+
except (NoOrigin, NoModel, HostError) as e:
|
|
492
|
+
return _refusal(e, request)
|
|
493
|
+
except OSError as e:
|
|
494
|
+
raise httpx.ConnectError(str(e), request=request) from e
|
|
495
|
+
for _ in range(MAX_TRIES):
|
|
496
|
+
try:
|
|
497
|
+
g = await self._wait(call, model, origin)
|
|
498
|
+
except (Refused, HostError) as e:
|
|
499
|
+
return _refusal(e, request)
|
|
500
|
+
except OSError as e:
|
|
501
|
+
raise httpx.ConnectError(str(e), request=request) from e
|
|
502
|
+
used = g.model or model
|
|
503
|
+
try:
|
|
504
|
+
resp = await self._inner.handle_async_request(_to_host(request, self._credential, call.body_for(used, g), g))
|
|
505
|
+
except BaseException:
|
|
506
|
+
self._release_later(g, used)
|
|
507
|
+
raise
|
|
508
|
+
if stopped(resp.headers):
|
|
509
|
+
await resp.aclose()
|
|
510
|
+
self._release_later(g, used)
|
|
511
|
+
continue
|
|
512
|
+
if _streams(resp):
|
|
513
|
+
return httpx.Response(resp.status_code, headers=resp.headers, request=request, extensions=resp.extensions,
|
|
514
|
+
stream=_AsyncGrantStream(resp, lambda r, w, g=g, used=used: self._release_later(g, used, r, w)))
|
|
515
|
+
raw = b""
|
|
516
|
+
try:
|
|
517
|
+
async for chunk in resp.aiter_raw():
|
|
518
|
+
raw += chunk
|
|
519
|
+
finally:
|
|
520
|
+
await resp.aclose()
|
|
521
|
+
self._release_later(g, used, *usage_of(raw))
|
|
522
|
+
return _whole(resp, raw, request)
|
|
523
|
+
return _error(503, "the waiting line stopped this call to make way for someone; try again", "nc_stopped",
|
|
524
|
+
request, retry_after=1)
|
|
525
|
+
|
|
526
|
+
async def aclose(self) -> None:
|
|
527
|
+
await self._inner.aclose()
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
# The transport under each client http_client made, for cancel.
|
|
531
|
+
_made: "weakref.WeakKeyDictionary[httpx.Client, HostTransport]" = weakref.WeakKeyDictionary()
|
|
532
|
+
_made_lock = threading.Lock()
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def http_client(launched: Optional[Launch] = None, **kwargs: Any) -> httpx.Client:
|
|
536
|
+
"""An httpx client whose every model call goes through the host's line:
|
|
537
|
+
OpenAI(base_url=BASE_URL, api_key=<anything>, http_client=http_client())."""
|
|
538
|
+
kwargs.setdefault("timeout", TIMEOUT)
|
|
539
|
+
t = HostTransport(launched)
|
|
540
|
+
c = httpx.Client(base_url=BASE_URL, transport=t, **kwargs)
|
|
541
|
+
with _made_lock:
|
|
542
|
+
_made[c] = t
|
|
543
|
+
return c
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def cancel(client: Any) -> bool:
|
|
547
|
+
"""Gives up the model calls a client of the kit's is making, from any
|
|
548
|
+
thread, at once (HostTransport.cancel): client is the OpenAI client made
|
|
549
|
+
with http_client=http_client(), that httpx client itself, or a
|
|
550
|
+
HostTransport. A call waiting in line leaves it; a call under way stops
|
|
551
|
+
at its engine, wherever the host placed it, and its grant goes back; the
|
|
552
|
+
thread making it gets Cancelled, which OpenAI's library raises as
|
|
553
|
+
APIConnectionError. The client takes no model call after: make a new
|
|
554
|
+
one for the next. False, doing nothing, for any other client (one not
|
|
555
|
+
on the kit's synchronous transport)."""
|
|
556
|
+
if isinstance(client, HostTransport):
|
|
557
|
+
client.cancel()
|
|
558
|
+
return True
|
|
559
|
+
with _made_lock:
|
|
560
|
+
for c in (client, getattr(client, "_client", None)): # OpenAI's client keeps its httpx client as _client
|
|
561
|
+
try:
|
|
562
|
+
t = _made.get(c) if c is not None else None
|
|
563
|
+
except TypeError:
|
|
564
|
+
t = None
|
|
565
|
+
if t is not None:
|
|
566
|
+
break
|
|
567
|
+
if t is None:
|
|
568
|
+
return False
|
|
569
|
+
t.cancel()
|
|
570
|
+
return True
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def async_http_client(launched: Optional[Launch] = None, **kwargs: Any) -> httpx.AsyncClient:
|
|
574
|
+
"""http_client for AsyncOpenAI."""
|
|
575
|
+
kwargs.setdefault("timeout", TIMEOUT)
|
|
576
|
+
return httpx.AsyncClient(base_url=BASE_URL, transport=AsyncHostTransport(launched), **kwargs)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nativecluster
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: NativeCluster's kit for Python apps: the app's half of the host API
|
|
5
|
+
Author: NativeCluster
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://www.nativecluster.com
|
|
8
|
+
Project-URL: Documentation, https://www.nativecluster.com/docs/apps#apps-your-group-builds
|
|
9
|
+
Keywords: nativecluster,local-llm,private-ai,self-hosted,openai
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: MacOS
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
16
|
+
Requires-Python: >=3.9
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Provides-Extra: openai
|
|
20
|
+
Requires-Dist: httpx>=0.23; extra == "openai"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# nativecluster
|
|
24
|
+
|
|
25
|
+
The Python kit for building [NativeCluster](https://www.nativecluster.com)
|
|
26
|
+
apps.
|
|
27
|
+
|
|
28
|
+
NativeCluster is a free Mac app that runs open AI models on a group's own
|
|
29
|
+
Macs. An app made for it is a program NativeCluster installs and runs for the
|
|
30
|
+
group's people: it serves its pages behind NativeCluster's sign-in, and every
|
|
31
|
+
model call it makes waits its turn in the cluster's one line and runs on one
|
|
32
|
+
of the group's own Macs. This kit is the app's side of that:
|
|
33
|
+
|
|
34
|
+
- **Serving:** a WSGI app (Flask, Django, a plain function) or an ASGI one
|
|
35
|
+
(FastAPI, Starlette) runs behind NativeCluster with one line,
|
|
36
|
+
`run_wsgi` or `run_asgi`, which checks NativeCluster's signed ticket on
|
|
37
|
+
every request.
|
|
38
|
+
- **People:** `person_of(request)` is the person NativeCluster signed in,
|
|
39
|
+
read from that ticket and nothing else.
|
|
40
|
+
- **Models:** `models().chat(<task>, messages)` asks the group's models
|
|
41
|
+
through the line. Or keep OpenAI's Python library and hand it
|
|
42
|
+
`transport.http_client()`.
|
|
43
|
+
- **Schedules and background work:** `host()` sets the app's schedules,
|
|
44
|
+
events and loops, and `run_firing` does the work when one comes due.
|
|
45
|
+
- **Data:** `data_dir()` is the app's own folder.
|
|
46
|
+
|
|
47
|
+
You don't need this kit to call a cluster's models from a script or tool.
|
|
48
|
+
Any OpenAI-compatible client works with NativeCluster's local address,
|
|
49
|
+
`http://localhost:8487/v1`: see [the API docs](https://www.nativecluster.com/docs/api).
|
|
50
|
+
|
|
51
|
+
## Install
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
pip install nativecluster
|
|
55
|
+
pip install "nativecluster[openai]" # for nativecluster.transport, with OpenAI's library
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Python 3.9 or newer. The kit itself needs nothing else.
|
|
59
|
+
|
|
60
|
+
`ncdev`, NativeCluster's tool for building apps, comes with NativeCluster.
|
|
61
|
+
`ncdev new <dir> -lang python` writes a whole example app with this kit
|
|
62
|
+
inside it, and `ncdev check` holds an app to NativeCluster's rules before it
|
|
63
|
+
is installed. An app's package carries its own Python and every dependency,
|
|
64
|
+
so install the kit into the Python the app ships with.
|
|
65
|
+
|
|
66
|
+
## A small app
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
import nativecluster as nc
|
|
70
|
+
|
|
71
|
+
def app(environ, start_response):
|
|
72
|
+
person = nc.person_of(environ)
|
|
73
|
+
answer = nc.models().chat("summarize", [
|
|
74
|
+
{"role": "user", "content": "Summarize: NativeCluster runs AI on our own Macs."},
|
|
75
|
+
])
|
|
76
|
+
start_response("200 OK", [("Content-Type", "text/plain; charset=utf-8")])
|
|
77
|
+
return [f"{person}: {answer.text} ({answer.model})".encode()]
|
|
78
|
+
|
|
79
|
+
if __name__ == "__main__":
|
|
80
|
+
nc.run_wsgi("nc-app.json", app)
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`summarize` is one of the tasks the app's manifest (`nc-app.json`) names.
|
|
84
|
+
NativeCluster decides which of the group's models may do it. The program
|
|
85
|
+
runs only under NativeCluster (`ncdev run` while you build it); started any
|
|
86
|
+
other way, it says so and stops.
|
|
87
|
+
|
|
88
|
+
With OpenAI's library, a call's `model` names one of the app's tasks:
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from openai import OpenAI
|
|
92
|
+
from nativecluster import transport
|
|
93
|
+
|
|
94
|
+
client = OpenAI(base_url=transport.BASE_URL, api_key="nativecluster",
|
|
95
|
+
http_client=transport.http_client())
|
|
96
|
+
client.chat.completions.create(model="summarize", messages=[...])
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Learn more
|
|
100
|
+
|
|
101
|
+
- [Apps your group builds](https://www.nativecluster.com/docs/apps#apps-your-group-builds)
|
|
102
|
+
- [NativeCluster](https://www.nativecluster.com)
|
|
103
|
+
|
|
104
|
+
## License
|
|
105
|
+
|
|
106
|
+
This kit is licensed under the Apache License, Version 2.0. NativeCluster
|
|
107
|
+
itself is proprietary.
|