nativecluster 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,576 @@
1
+ """OpenAI's Python library, through the host.
2
+
3
+ An app built on OpenAI's library (an agent ported to NativeCluster) keeps
4
+ it: it hands the library this module's HTTP client, and every model call the
5
+ library makes waits its turn in the host's one line as whatever started it
6
+ (the request being handled, a firing, a job: the kit's origin), runs on the
7
+ grant the line gave it, and gives the grant back when its answer ends, read
8
+ to the end or not. A call's "model" names one of the app's tasks, never a
9
+ model: the kit asks the host which of the group's models may do it, and the
10
+ host sends the call to that one. The app never reaches the engine. An app
11
+ that lets a person pick among them (a model menu) names the pick after the
12
+ task, "<task>/<model>", one of the models the host names for that task
13
+ (Models.choices); a model it no longer names gives way to the task's
14
+ first, so a pick never breaks a call.
15
+
16
+ client = OpenAI(base_url=transport.BASE_URL, api_key="nativecluster",
17
+ http_client=transport.http_client())
18
+ client.chat.completions.create(model="<one of the app's tasks>", messages=[...])
19
+
20
+ What the line says comes back in OpenAI's shapes, so the library's own
21
+ handling applies: a call the line stopped to make way for someone is run
22
+ again (a stream already under way ends with an error frame instead); a
23
+ prompt longer than the model's window is OpenAI's context_length_exceeded;
24
+ no origin is a 403, no model for the task a 404 (model_not_found), a refused
25
+ wait a 503. It needs httpx, which OpenAI's library carries.
26
+
27
+ An app that stops a call from another thread (its person closed the window
28
+ while the agent's thread waits in line or reads the answer) gives the call
29
+ up with cancel(client): the wait leaves the line, the grant goes back, the
30
+ model stops writing wherever the host placed the call, and the thread
31
+ making the call gets Cancelled.
32
+
33
+ transport.cancel(client) # the OpenAI client, or http_client()'s
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ import asyncio
39
+ import json
40
+ import threading
41
+ import weakref
42
+ from typing import Any, AsyncIterator, Callable, Iterator, Optional, Union
43
+
44
+ import httpx
45
+
46
+ from ._hostsock import BASE_URL as _HOST_URL
47
+ from ._hostsock import HostError, HostSocket
48
+ from ._launch import Launch, launch
49
+ from ._models import (
50
+ GRANT_HEADER,
51
+ KIND_CHAT,
52
+ KIND_TRANSCRIPTION,
53
+ MAX_TRIES,
54
+ ROUTES,
55
+ EventScan,
56
+ Grant,
57
+ Models,
58
+ NoModel,
59
+ NoOrigin,
60
+ Refused,
61
+ Waiting,
62
+ multipart_field,
63
+ overflow_body,
64
+ stopped,
65
+ usage_of,
66
+ )
67
+ from ._origin import Origin
68
+
69
+ # BASE_URL is the base URL to give OpenAI's library: the host's socket is the
70
+ # destination, whatever the URL says.
71
+ BASE_URL = _HOST_URL + "/v1"
72
+
73
+ # TIMEOUT waits as long as an answer takes to read (a long answer streams
74
+ # for minutes), and bounds the rest. Waiting in line is not counted in it.
75
+ TIMEOUT = httpx.Timeout(connect=15.0, read=None, write=15.0, pool=10.0)
76
+
77
+ _EARLY = (
78
+ b'data: {"error": {"message": "the answer ended early: the waiting line may have stopped it to make way for '
79
+ b'someone; try again", "type": "server_error", "code": "nc_stopped"}}\n\n'
80
+ )
81
+
82
+
83
+ class Cancelled(httpx.TransportError):
84
+ """A model call cancel() gave up: it left the line, or its grant went
85
+ back and its answer ended. OpenAI's library reads it as a connection
86
+ error (APIConnectionError), and every retry of it on the same client is
87
+ refused at once, so nothing waits in line again."""
88
+
89
+
90
+ def _cancelled(request: Optional[httpx.Request] = None) -> Cancelled:
91
+ return Cancelled("nativecluster: this model call was given up (transport.cancel)", request=request)
92
+
93
+
94
+ def _later(fn: Callable[..., None], *args: Any) -> None:
95
+ """fn(*args) on a thread of its own: a give-back from cancel never
96
+ waits for the host, and happens even while the caller unwinds."""
97
+ threading.Thread(target=fn, args=args, daemon=True).start()
98
+
99
+
100
+ class _Held:
101
+ """The grant one call holds, given back once: by the call as its
102
+ answer ends (with what it did), or by cancel."""
103
+
104
+ def __init__(self, models: Models, grant: Grant, model: str) -> None:
105
+ self._models = models
106
+ self.grant = grant
107
+ self.model = model
108
+ self._lock = threading.Lock()
109
+ self._given = False
110
+
111
+ def give_back(self, read: int = 0, written: int = 0) -> None:
112
+ with self._lock:
113
+ if self._given:
114
+ return
115
+ self._given = True
116
+ self._models.release(self.grant, self.model, read, written)
117
+
118
+
119
+ def _error(status: int, message: str, code: str, request: httpx.Request, retry_after: int = 0) -> httpx.Response:
120
+ headers = {"Retry-After": str(retry_after)} if retry_after else {}
121
+ body = {"error": {"message": message, "type": "invalid_request_error" if status < 500 else "server_error", "code": code}}
122
+ return httpx.Response(status, json=body, headers=headers, request=request)
123
+
124
+
125
+ class _Call:
126
+ """One model call's plan: its kind, the task its "model" names, and its
127
+ body made for the model and the turn the host gave it."""
128
+
129
+ def __init__(self, kind: str, model: str, body_for: Callable[[str, Grant], bytes]) -> None:
130
+ self.kind = kind
131
+ # "<task>" or "<task>/<model>": a task's id holds no "/", nor does a
132
+ # catalog model's name.
133
+ self.task, _, self.want = model.partition("/")
134
+ self.body_for = body_for
135
+
136
+
137
+ def _plan(request: httpx.Request, body: bytes) -> Union[_Call, httpx.Response, None]:
138
+ """request's plan, an error to answer at once, or None for a request
139
+ that is no model call (it goes to the host as it is)."""
140
+ kind = ROUTES.get(request.url.path.rstrip("/"))
141
+ if kind is None or request.method != "POST":
142
+ return None
143
+ if kind == KIND_TRANSCRIPTION:
144
+ ctype = request.headers.get("content-type", "")
145
+ _, task = multipart_field(body, ctype, "model")
146
+ if not task:
147
+ return _error(400, "a model call names one of the app's tasks as its model", "nc_no_task", request)
148
+ return _Call(kind, task, lambda model, g: multipart_field(body, ctype, "model", model)[0])
149
+ try:
150
+ obj = json.loads(body)
151
+ except ValueError:
152
+ obj = None
153
+ if not isinstance(obj, dict) or not isinstance(obj.get("model"), str) or not obj["model"]:
154
+ return _error(400, "a model call names one of the app's tasks as its model", "nc_no_task", request)
155
+
156
+ def body_for(model: str, g: Grant) -> bytes:
157
+ b = dict(obj)
158
+ b["model"] = model
159
+ if kind == KIND_CHAT and g.cache_key:
160
+ b["cache_key"] = g.cache_key
161
+ return json.dumps(b).encode("utf-8")
162
+
163
+ return _Call(kind, obj["model"], body_for)
164
+
165
+
166
+ def _to_host(request: httpx.Request, credential: str, content: Optional[bytes] = None, grant: Optional[Grant] = None) -> httpx.Request:
167
+ """request, sent to the host with the app's credential (whatever key the
168
+ library sent) and the grant it runs on."""
169
+ headers = [(k, v) for k, v in request.headers.multi_items()
170
+ if k.lower() not in ("authorization", "content-length", "transfer-encoding", "host", GRANT_HEADER.lower())]
171
+ headers.append(("Authorization", "Bearer " + credential))
172
+ if grant is not None:
173
+ headers.append((GRANT_HEADER, grant.id))
174
+ return httpx.Request(request.method, request.url, headers=headers,
175
+ content=request.content if content is None else content, extensions=request.extensions)
176
+
177
+
178
+ def _streams(resp: httpx.Response) -> bool:
179
+ return resp.status_code == 200 and resp.headers.get("content-type", "").startswith("text/event-stream")
180
+
181
+
182
+ def _whole(resp: httpx.Response, raw: bytes, request: httpx.Request) -> httpx.Response:
183
+ headers = [(k, v) for k, v in resp.headers.multi_items() if k.lower() not in ("content-length", "transfer-encoding")]
184
+ return httpx.Response(resp.status_code, headers=headers, content=overflow_body(resp.status_code, raw),
185
+ request=request, extensions=resp.extensions)
186
+
187
+
188
+ def _refusal(e: Exception, request: httpx.Request) -> httpx.Response:
189
+ if isinstance(e, NoOrigin):
190
+ return _error(403, str(e), "nc_no_origin", request)
191
+ if isinstance(e, NoModel):
192
+ return _error(404, str(e), "model_not_found", request)
193
+ if isinstance(e, HostError):
194
+ words = _host_words(e)
195
+ if words:
196
+ # NativeCluster's own words to the person (this computer's switch
197
+ # is off, it is not added to the cluster or was removed, Members
198
+ # does not give them the app): a refusal to show as it is.
199
+ return _error(503, words, "nc_refused", request)
200
+ return _error(e.status if e.status >= 400 else 502, str(e), "nc_host", request)
201
+ if isinstance(e, Refused) and e.words:
202
+ # The line's own words for the person, as they are.
203
+ return _error(503, e.words, "nc_refused", request)
204
+ return _error(503, str(e), "nc_refused", request)
205
+
206
+
207
+ # The host's codes for an answer whose error is words for the person.
208
+ _PERSON_CODES = ("inference_off", "computer_waiting", "computer_removed", "not_given", "app_not_here")
209
+
210
+
211
+ def _host_words(e: "HostError") -> str:
212
+ try:
213
+ body = json.loads(e.text)
214
+ except ValueError:
215
+ return ""
216
+ if isinstance(body, dict) and body.get("code") in _PERSON_CODES and isinstance(body.get("error"), str):
217
+ return body["error"]
218
+ return ""
219
+
220
+
221
+ class _GrantStream(httpx.SyncByteStream):
222
+ """A streamed answer passed on as it comes, its grant given back once it
223
+ ends: read to the end, or closed early. cancelled says whether cancel()
224
+ gave the call up: its answer, ended by the host, raises Cancelled rather
225
+ than reading as a stream the line stopped."""
226
+
227
+ def __init__(self, resp: httpx.Response, give_back: Callable[[int, int], None],
228
+ cancelled: Optional[Callable[[], bool]] = None) -> None:
229
+ self._resp = resp
230
+ self._give_back = give_back
231
+ self._cancelled = cancelled or (lambda: False)
232
+ self._scan = EventScan()
233
+ self._lock = threading.Lock()
234
+ self._given = False
235
+
236
+ def _end(self) -> None:
237
+ with self._lock:
238
+ if self._given:
239
+ return
240
+ self._given = True
241
+ self._give_back(self._scan.read, self._scan.written)
242
+
243
+ def __iter__(self) -> Iterator[bytes]:
244
+ try:
245
+ for chunk in self._resp.iter_raw():
246
+ out, _ = self._scan.feed(chunk)
247
+ if out:
248
+ yield out
249
+ except Exception as e:
250
+ if self._cancelled():
251
+ self._end()
252
+ raise _cancelled() from e
253
+ raise
254
+ out, _ = self._scan.flush()
255
+ if out:
256
+ yield out
257
+ self._end()
258
+ if not self._scan.done and self._scan.error is None:
259
+ if self._cancelled():
260
+ raise _cancelled()
261
+ yield _EARLY
262
+
263
+ def close(self) -> None:
264
+ try:
265
+ self._resp.close()
266
+ finally:
267
+ self._end()
268
+
269
+
270
+ class _AsyncGrantStream(httpx.AsyncByteStream):
271
+ def __init__(self, resp: httpx.Response, give_back: Callable[[int, int], None]) -> None:
272
+ self._resp = resp
273
+ self._give_back = give_back
274
+ self._scan = EventScan()
275
+ self._lock = threading.Lock()
276
+ self._given = False
277
+
278
+ def _end(self) -> None:
279
+ with self._lock:
280
+ if self._given:
281
+ return
282
+ self._given = True
283
+ self._give_back(self._scan.read, self._scan.written)
284
+
285
+ async def __aiter__(self) -> AsyncIterator[bytes]:
286
+ async for chunk in self._resp.aiter_raw():
287
+ out, _ = self._scan.feed(chunk)
288
+ if out:
289
+ yield out
290
+ out, _ = self._scan.flush()
291
+ if out:
292
+ yield out
293
+ self._end()
294
+ if not self._scan.done and self._scan.error is None:
295
+ yield _EARLY
296
+
297
+ async def aclose(self) -> None:
298
+ try:
299
+ await self._resp.aclose()
300
+ finally:
301
+ self._end()
302
+
303
+
304
+ def _models_for(launched: Optional[Launch], models: Optional[Models]) -> "tuple[Launch, Models]":
305
+ l = launched or launch()
306
+ return l, models or Models(HostSocket(l.host_socket, l.credential), test=l.test)
307
+
308
+
309
+ class HostTransport(httpx.BaseTransport):
310
+ """httpx's transport for OpenAI's library behind the host (sync)."""
311
+
312
+ def __init__(self, launched: Optional[Launch] = None, *, models: Optional[Models] = None) -> None:
313
+ l, self._models = _models_for(launched, models)
314
+ self._credential = l.credential
315
+ self._inner = httpx.HTTPTransport(uds=l.host_socket)
316
+ # What cancel gives up: the waits in line now and the grants calls
317
+ # hold now.
318
+ self._lock = threading.Lock()
319
+ self._cancelled = False
320
+ self._waits: "set[Waiting]" = set()
321
+ self._held: "set[_Held]" = set()
322
+
323
+ def cancel(self) -> None:
324
+ """Gives up every model call this transport is making, and every
325
+ one made on it after: a cancelled transport is done, so a retry
326
+ never waits in line again (make a new client for new calls). Safe
327
+ from any thread, and returns at once, without waiting for the host:
328
+
329
+ - a call still waiting in line leaves it: its held wait's
330
+ connection is shut down (never closed under its thread), and the
331
+ host gives its place away;
332
+ - a call that holds a grant gives it back (POST
333
+ /host/v1/line/release, on a thread of its own): the host stops
334
+ the call at its engine, on this Mac or wherever it placed it, and
335
+ ends its answer;
336
+ - the thread making the call gets Cancelled, whether it waited in
337
+ line, waited for the answer or read the stream.
338
+
339
+ Calls that are no model call (the models list) are not touched."""
340
+ with self._lock:
341
+ self._cancelled = True
342
+ waits, self._waits = self._waits, set()
343
+ held, self._held = self._held, set()
344
+ # Under the lock, so a wait answered meanwhile finds itself
345
+ # given up and leaves its grant to this (_wait).
346
+ unused = [w.cancel() for w in waits]
347
+ for g in unused:
348
+ if g is not None:
349
+ _later(self._models.give_back, g)
350
+ for h in held:
351
+ _later(h.give_back)
352
+
353
+ @property
354
+ def cancelled(self) -> bool:
355
+ """Whether cancel() gave this transport's calls up."""
356
+ return self._cancelled
357
+
358
+ def _wait(self, call: _Call, model: str, origin: Origin, request: httpx.Request) -> Union[_Held, httpx.Response]:
359
+ """One wait in line for call, which cancel can give up: the grant it
360
+ got, held, or the line's refusal to answer with."""
361
+ waiting = Waiting()
362
+ with self._lock:
363
+ if self._cancelled:
364
+ raise _cancelled(request)
365
+ self._waits.add(waiting)
366
+ try:
367
+ g = self._models.wait(call.task, model=model, pick_chat=call.kind == KIND_CHAT, origin=origin,
368
+ waiting=waiting)
369
+ except (Refused, HostError) as e:
370
+ with self._lock:
371
+ self._waits.discard(waiting)
372
+ if self._cancelled:
373
+ raise _cancelled(request) from e
374
+ return _refusal(e, request)
375
+ except OSError as e:
376
+ with self._lock:
377
+ self._waits.discard(waiting)
378
+ if self._cancelled:
379
+ raise _cancelled(request) from e
380
+ raise httpx.ConnectError(str(e), request=request) from e
381
+ held = _Held(self._models, g, g.model or model)
382
+ with self._lock:
383
+ self._waits.discard(waiting)
384
+ given_up = waiting.cancelled
385
+ if not given_up:
386
+ waiting.take()
387
+ self._held.add(held)
388
+ if given_up:
389
+ # cancel reached the wait as it was answered: cancel gives the
390
+ # grant back.
391
+ raise _cancelled(request)
392
+ return held
393
+
394
+ def _give_back(self, held: _Held, read: int = 0, written: int = 0) -> None:
395
+ with self._lock:
396
+ self._held.discard(held)
397
+ held.give_back(read, written)
398
+
399
+ def handle_request(self, request: httpx.Request) -> httpx.Response:
400
+ call = _plan(request, request.read())
401
+ if call is None:
402
+ return self._inner.handle_request(_to_host(request, self._credential))
403
+ if isinstance(call, httpx.Response):
404
+ return call
405
+ try:
406
+ origin = self._models.origin()
407
+ model = self._models.model_for(call.task, call.kind, call.want)
408
+ except (NoOrigin, NoModel, HostError) as e:
409
+ return _refusal(e, request)
410
+ except OSError as e:
411
+ raise httpx.ConnectError(str(e), request=request) from e
412
+ for _ in range(MAX_TRIES):
413
+ held = self._wait(call, model, origin, request)
414
+ if isinstance(held, httpx.Response):
415
+ return held
416
+ g, used = held.grant, held.model
417
+ try:
418
+ resp = self._inner.handle_request(_to_host(request, self._credential, call.body_for(used, g), g))
419
+ except BaseException as e:
420
+ self._give_back(held)
421
+ if self._cancelled:
422
+ raise _cancelled(request) from e
423
+ raise
424
+ if stopped(resp.headers):
425
+ resp.close()
426
+ self._give_back(held)
427
+ continue
428
+ if _streams(resp):
429
+ return httpx.Response(resp.status_code, headers=resp.headers, request=request, extensions=resp.extensions,
430
+ stream=_GrantStream(resp, lambda r, w, held=held: self._give_back(held, r, w),
431
+ cancelled=lambda: self._cancelled))
432
+ raw = b""
433
+ try:
434
+ for chunk in resp.iter_raw():
435
+ raw += chunk
436
+ except Exception as e:
437
+ if self._cancelled:
438
+ raise _cancelled(request) from e
439
+ raise
440
+ finally:
441
+ resp.close()
442
+ self._give_back(held, *usage_of(raw))
443
+ if self._cancelled:
444
+ raise _cancelled(request)
445
+ return _whole(resp, raw, request)
446
+ return _error(503, "the waiting line stopped this call to make way for someone; try again", "nc_stopped",
447
+ request, retry_after=1)
448
+
449
+ def close(self) -> None:
450
+ self._inner.close()
451
+
452
+
453
+ class AsyncHostTransport(httpx.AsyncBaseTransport):
454
+ """httpx's transport for OpenAI's library behind the host (asyncio).
455
+ The kit's waits in line run on threads, so the event loop never waits
456
+ for one; a call cancelled while it waits gives its place in line up."""
457
+
458
+ def __init__(self, launched: Optional[Launch] = None, *, models: Optional[Models] = None) -> None:
459
+ l, self._models = _models_for(launched, models)
460
+ self._credential = l.credential
461
+ self._inner = httpx.AsyncHTTPTransport(uds=l.host_socket)
462
+
463
+ def _release_later(self, g: Grant, model: str, read: int = 0, written: int = 0) -> None:
464
+ # On a thread of its own: a release must happen even while the
465
+ # caller is being cancelled.
466
+ threading.Thread(target=self._models.release, args=(g, model, read, written), daemon=True).start()
467
+
468
+ async def _wait(self, call: _Call, model: str, origin: Origin) -> Grant:
469
+ waiting = Waiting()
470
+ try:
471
+ await asyncio.to_thread(self._models.wait, call.task, model=model, pick_chat=call.kind == KIND_CHAT,
472
+ origin=origin, waiting=waiting)
473
+ except asyncio.CancelledError:
474
+ left = waiting.cancel()
475
+ if left is not None:
476
+ self._release_later(left, left.model or model)
477
+ raise
478
+ g = waiting.take()
479
+ assert g is not None
480
+ return g
481
+
482
+ async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
483
+ call = _plan(request, await request.aread())
484
+ if call is None:
485
+ return await self._inner.handle_async_request(_to_host(request, self._credential))
486
+ if isinstance(call, httpx.Response):
487
+ return call
488
+ try:
489
+ origin = self._models.origin()
490
+ model = await asyncio.to_thread(self._models.model_for, call.task, call.kind, call.want)
491
+ except (NoOrigin, NoModel, HostError) as e:
492
+ return _refusal(e, request)
493
+ except OSError as e:
494
+ raise httpx.ConnectError(str(e), request=request) from e
495
+ for _ in range(MAX_TRIES):
496
+ try:
497
+ g = await self._wait(call, model, origin)
498
+ except (Refused, HostError) as e:
499
+ return _refusal(e, request)
500
+ except OSError as e:
501
+ raise httpx.ConnectError(str(e), request=request) from e
502
+ used = g.model or model
503
+ try:
504
+ resp = await self._inner.handle_async_request(_to_host(request, self._credential, call.body_for(used, g), g))
505
+ except BaseException:
506
+ self._release_later(g, used)
507
+ raise
508
+ if stopped(resp.headers):
509
+ await resp.aclose()
510
+ self._release_later(g, used)
511
+ continue
512
+ if _streams(resp):
513
+ return httpx.Response(resp.status_code, headers=resp.headers, request=request, extensions=resp.extensions,
514
+ stream=_AsyncGrantStream(resp, lambda r, w, g=g, used=used: self._release_later(g, used, r, w)))
515
+ raw = b""
516
+ try:
517
+ async for chunk in resp.aiter_raw():
518
+ raw += chunk
519
+ finally:
520
+ await resp.aclose()
521
+ self._release_later(g, used, *usage_of(raw))
522
+ return _whole(resp, raw, request)
523
+ return _error(503, "the waiting line stopped this call to make way for someone; try again", "nc_stopped",
524
+ request, retry_after=1)
525
+
526
+ async def aclose(self) -> None:
527
+ await self._inner.aclose()
528
+
529
+
530
+ # The transport under each client http_client made, for cancel.
531
+ _made: "weakref.WeakKeyDictionary[httpx.Client, HostTransport]" = weakref.WeakKeyDictionary()
532
+ _made_lock = threading.Lock()
533
+
534
+
535
+ def http_client(launched: Optional[Launch] = None, **kwargs: Any) -> httpx.Client:
536
+ """An httpx client whose every model call goes through the host's line:
537
+ OpenAI(base_url=BASE_URL, api_key=<anything>, http_client=http_client())."""
538
+ kwargs.setdefault("timeout", TIMEOUT)
539
+ t = HostTransport(launched)
540
+ c = httpx.Client(base_url=BASE_URL, transport=t, **kwargs)
541
+ with _made_lock:
542
+ _made[c] = t
543
+ return c
544
+
545
+
546
+ def cancel(client: Any) -> bool:
547
+ """Gives up the model calls a client of the kit's is making, from any
548
+ thread, at once (HostTransport.cancel): client is the OpenAI client made
549
+ with http_client=http_client(), that httpx client itself, or a
550
+ HostTransport. A call waiting in line leaves it; a call under way stops
551
+ at its engine, wherever the host placed it, and its grant goes back; the
552
+ thread making it gets Cancelled, which OpenAI's library raises as
553
+ APIConnectionError. The client takes no model call after: make a new
554
+ one for the next. False, doing nothing, for any other client (one not
555
+ on the kit's synchronous transport)."""
556
+ if isinstance(client, HostTransport):
557
+ client.cancel()
558
+ return True
559
+ with _made_lock:
560
+ for c in (client, getattr(client, "_client", None)): # OpenAI's client keeps its httpx client as _client
561
+ try:
562
+ t = _made.get(c) if c is not None else None
563
+ except TypeError:
564
+ t = None
565
+ if t is not None:
566
+ break
567
+ if t is None:
568
+ return False
569
+ t.cancel()
570
+ return True
571
+
572
+
573
+ def async_http_client(launched: Optional[Launch] = None, **kwargs: Any) -> httpx.AsyncClient:
574
+ """http_client for AsyncOpenAI."""
575
+ kwargs.setdefault("timeout", TIMEOUT)
576
+ return httpx.AsyncClient(base_url=BASE_URL, transport=AsyncHostTransport(launched), **kwargs)
@@ -0,0 +1,107 @@
1
+ Metadata-Version: 2.4
2
+ Name: nativecluster
3
+ Version: 0.1.0
4
+ Summary: NativeCluster's kit for Python apps: the app's half of the host API
5
+ Author: NativeCluster
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Homepage, https://www.nativecluster.com
8
+ Project-URL: Documentation, https://www.nativecluster.com/docs/apps#apps-your-group-builds
9
+ Keywords: nativecluster,local-llm,private-ai,self-hosted,openai
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Operating System :: MacOS
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3 :: Only
15
+ Classifier: Topic :: Software Development :: Libraries
16
+ Requires-Python: >=3.9
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Provides-Extra: openai
20
+ Requires-Dist: httpx>=0.23; extra == "openai"
21
+ Dynamic: license-file
22
+
23
+ # nativecluster
24
+
25
+ The Python kit for building [NativeCluster](https://www.nativecluster.com)
26
+ apps.
27
+
28
+ NativeCluster is a free Mac app that runs open AI models on a group's own
29
+ Macs. An app made for it is a program NativeCluster installs and runs for the
30
+ group's people: it serves its pages behind NativeCluster's sign-in, and every
31
+ model call it makes waits its turn in the cluster's one line and runs on one
32
+ of the group's own Macs. This kit is the app's side of that:
33
+
34
+ - **Serving:** a WSGI app (Flask, Django, a plain function) or an ASGI one
35
+ (FastAPI, Starlette) runs behind NativeCluster with one line,
36
+ `run_wsgi` or `run_asgi`, which checks NativeCluster's signed ticket on
37
+ every request.
38
+ - **People:** `person_of(request)` is the person NativeCluster signed in,
39
+ read from that ticket and nothing else.
40
+ - **Models:** `models().chat(<task>, messages)` asks the group's models
41
+ through the line. Or keep OpenAI's Python library and hand it
42
+ `transport.http_client()`.
43
+ - **Schedules and background work:** `host()` sets the app's schedules,
44
+ events and loops, and `run_firing` does the work when one comes due.
45
+ - **Data:** `data_dir()` is the app's own folder.
46
+
47
+ You don't need this kit to call a cluster's models from a script or tool.
48
+ Any OpenAI-compatible client works with NativeCluster's local address,
49
+ `http://localhost:8487/v1`: see [the API docs](https://www.nativecluster.com/docs/api).
50
+
51
+ ## Install
52
+
53
+ ```sh
54
+ pip install nativecluster
55
+ pip install "nativecluster[openai]" # for nativecluster.transport, with OpenAI's library
56
+ ```
57
+
58
+ Python 3.9 or newer. The kit itself needs nothing else.
59
+
60
+ `ncdev`, NativeCluster's tool for building apps, comes with NativeCluster.
61
+ `ncdev new <dir> -lang python` writes a whole example app with this kit
62
+ inside it, and `ncdev check` holds an app to NativeCluster's rules before it
63
+ is installed. An app's package carries its own Python and every dependency,
64
+ so install the kit into the Python the app ships with.
65
+
66
+ ## A small app
67
+
68
+ ```python
69
+ import nativecluster as nc
70
+
71
+ def app(environ, start_response):
72
+ person = nc.person_of(environ)
73
+ answer = nc.models().chat("summarize", [
74
+ {"role": "user", "content": "Summarize: NativeCluster runs AI on our own Macs."},
75
+ ])
76
+ start_response("200 OK", [("Content-Type", "text/plain; charset=utf-8")])
77
+ return [f"{person}: {answer.text} ({answer.model})".encode()]
78
+
79
+ if __name__ == "__main__":
80
+ nc.run_wsgi("nc-app.json", app)
81
+ ```
82
+
83
+ `summarize` is one of the tasks the app's manifest (`nc-app.json`) names.
84
+ NativeCluster decides which of the group's models may do it. The program
85
+ runs only under NativeCluster (`ncdev run` while you build it); started any
86
+ other way, it says so and stops.
87
+
88
+ With OpenAI's library, a call's `model` names one of the app's tasks:
89
+
90
+ ```python
91
+ from openai import OpenAI
92
+ from nativecluster import transport
93
+
94
+ client = OpenAI(base_url=transport.BASE_URL, api_key="nativecluster",
95
+ http_client=transport.http_client())
96
+ client.chat.completions.create(model="summarize", messages=[...])
97
+ ```
98
+
99
+ ## Learn more
100
+
101
+ - [Apps your group builds](https://www.nativecluster.com/docs/apps#apps-your-group-builds)
102
+ - [NativeCluster](https://www.nativecluster.com)
103
+
104
+ ## License
105
+
106
+ This kit is licensed under the Apache License, Version 2.0. NativeCluster
107
+ itself is proprietary.