ommlds 0.0.0.dev503__py3-none-any.whl → 0.0.0.dev505__py3-none-any.whl

ommlds/minichain/backends/impls/groq/stream.py

@@ -6,21 +6,18 @@ from omlish import typedvalues as tv
 from omlish.formats import json
 from omlish.http import all as http
 from omlish.http import sse
-from omlish.io.buffers import DelimitingBuffer
 
 from .....backends.groq import protocol as pt
 from .....backends.groq.clients import REQUIRED_HTTP_HEADERS
-from ....chat.choices.services import ChatChoicesOutputs
 from ....chat.choices.stream.services import ChatChoicesStreamRequest
 from ....chat.choices.stream.services import ChatChoicesStreamResponse
 from ....chat.choices.stream.services import static_check_is_chat_choices_stream_service
 from ....chat.choices.stream.types import AiChoicesDeltas
 from ....chat.tools.types import Tool
 from ....configs import Config
-from ....resources import UseResources
+from ....http.stream import BytesHttpStreamResponseBuilder
+from ....http.stream import SimpleSseLinesHttpStreamResponseHandler
 from ....standard import ApiKey
-from ....stream.services import StreamResponseSink
-from ....stream.services import new_stream_response
 from .chat import GroqChatChoicesService
 from .names import MODEL_NAMES
 from .protocol import build_gq_request_messages
@@ -50,6 +47,35 @@ class GroqChatChoicesStreamService:
             self._model_name = cc.pop(GroqChatChoicesService.DEFAULT_MODEL_NAME)
             self._api_key = ApiKey.pop_secret(cc, env='GROQ_API_KEY')
 
+    URL: ta.ClassVar[str] = 'https://api.groq.com/openai/v1/chat/completions'
+
+    def _process_sse(self, so: sse.SseDecoderOutput) -> ta.Sequence[AiChoicesDeltas | None]:
+        if not (isinstance(so, sse.SseEvent) and so.type == b'message'):
+            return []
+
+        ss = so.data.decode('utf-8')
+        if ss == '[DONE]':
+            return [None]
+
+        sj = json.loads(ss)  # ChatCompletionChunk
+
+        check.state(sj['object'] == 'chat.completion.chunk')
+
+        ccc = msh.unmarshal(sj, pt.ChatCompletionChunk)
+
+        # FIXME: stop reason
+        if not ccc.choices:
+            return []
+
+        if any(choice.finish_reason for choice in ccc.choices):
+            check.state(all(choice.finish_reason for choice in ccc.choices))
+            return [None]
+
+        return [AiChoicesDeltas([
+            build_mc_ai_choice_deltas(choice.delta)
+            for choice in ccc.choices
+        ])]
+
     READ_CHUNK_SIZE: ta.ClassVar[int] = -1
 
     async def invoke(self, request: ChatChoicesStreamRequest) -> ChatChoicesStreamResponse:
@@ -69,7 +95,7 @@ class GroqChatChoicesStreamService:
         raw_request = msh.marshal(gq_request)
 
         http_request = http.HttpRequest(
-            'https://api.groq.com/openai/v1/chat/completions',
+            self.URL,
             headers={
                 http.consts.HEADER_CONTENT_TYPE: http.consts.CONTENT_TYPE_JSON,
                 http.consts.HEADER_AUTH: http.consts.format_bearer_auth_header(check.not_none(self._api_key).reveal()),
@@ -78,50 +104,11 @@ class GroqChatChoicesStreamService:
             data=json.dumps(raw_request).encode('utf-8'),
         )
 
-        async with UseResources.or_new(request.options) as rs:
-            http_client = await rs.enter_async_context(http.manage_async_client(self._http_client))
-            http_response = await rs.enter_async_context(await http_client.stream_request(http_request))
-
-            async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs]:
-                db = DelimitingBuffer([b'\r', b'\n', b'\r\n'])
-                sd = sse.SseDecoder()
-                while True:
-                    b = await http_response.stream.read1(self.READ_CHUNK_SIZE)
-                    for l in db.feed(b):
-                        if isinstance(l, DelimitingBuffer.Incomplete):
-                            # FIXME: handle
-                            raise TypeError(l)
-
-                        # FIXME: https://platform.openai.com/docs/guides/function-calling?api-mode=responses#streaming
-                        for so in sd.process_line(l):
-                            if isinstance(so, sse.SseEvent) and so.type == b'message':
-                                ss = so.data.decode('utf-8')
-                                if ss == '[DONE]':
-                                    return []
-
-                                sj = json.loads(ss)  # ChatCompletionChunk
-
-                                check.state(sj['object'] == 'chat.completion.chunk')
-
-                                ccc = msh.unmarshal(sj, pt.ChatCompletionChunk)
-
-                                # FIXME: stop reason
-                                if not ccc.choices:
-                                    continue
-
-                                if any(choice.finish_reason for choice in ccc.choices):
-                                    check.state(all(choice.finish_reason for choice in ccc.choices))
-                                    break
-
-                                await sink.emit(AiChoicesDeltas([
-                                    build_mc_ai_choice_deltas(choice.delta)
-                                    for choice in ccc.choices
-                                ]))
-
-                    if not b:
-                        return []
-
-            # raw_response = json.loads(check.not_none(http_response.data).decode('utf-8'))
-            # return rh.build_response(raw_response)
-
-            return await new_stream_response(rs, inner)
+        return await BytesHttpStreamResponseBuilder(
+            self._http_client,
+            lambda http_response: SimpleSseLinesHttpStreamResponseHandler(self._process_sse).as_lines().as_bytes(),
+            read_chunk_size=self.READ_CHUNK_SIZE,
+        ).new_stream_response(
+            http_request,
+            request.options,
+        )

ommlds/minichain/backends/impls/openai/stream.py

@@ -4,16 +4,13 @@ https://platform.openai.com/docs/api-reference/responses-streaming
 import typing as ta
 
 from omlish import check
-from omlish import dataclasses as dc
 from omlish import marshal as msh
 from omlish import typedvalues as tv
 from omlish.formats import json
 from omlish.http import all as http
 from omlish.http import sse
-from omlish.io.buffers import DelimitingBuffer
 
 from .....backends.openai import protocol as pt
-from ....chat.choices.services import ChatChoicesOutputs
 from ....chat.choices.stream.services import ChatChoicesStreamRequest
 from ....chat.choices.stream.services import ChatChoicesStreamResponse
 from ....chat.choices.stream.services import static_check_is_chat_choices_stream_service
@@ -21,12 +18,11 @@ from ....chat.choices.stream.types import AiChoiceDeltas
 from ....chat.choices.stream.types import AiChoicesDeltas
 from ....chat.choices.stream.types import ChatChoicesStreamOption
 from ....configs import Config
+from ....http.stream import BytesHttpStreamResponseBuilder
+from ....http.stream import SimpleSseLinesHttpStreamResponseHandler
 from ....resources import ResourcesOption
-from ....resources import UseResources
 from ....standard import ApiKey
 from ....stream.services import StreamOption
-from ....stream.services import StreamResponseSink
-from ....stream.services import new_stream_response
 from .chat import OpenaiChatChoicesService
 from .format import OpenaiChatRequestHandler
 from .format import build_mc_ai_delta
@@ -36,12 +32,6 @@ from .names import CHAT_MODEL_NAMES
 ##
 
 
-@dc.dataclass()
-class OpenaiChatChoicesStreamServiceError(Exception):
-    status: int
-    data: ta.Any | None = None
-
-
 # @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
 #     name='openai',
 #     type='ChatChoicesStreamService',
@@ -61,11 +51,38 @@ class OpenaiChatChoicesStreamService:
             self._model_name = cc.pop(OpenaiChatChoicesService.DEFAULT_MODEL_NAME)
             self._api_key = ApiKey.pop_secret(cc, env='OPENAI_API_KEY')
 
+    URL: ta.ClassVar[str] = 'https://api.openai.com/v1/chat/completions'
+
+    def _process_sse(self, so: sse.SseDecoderOutput) -> ta.Sequence[AiChoicesDeltas | None]:
+        if not (isinstance(so, sse.SseEvent) and so.type == b'message'):
+            return []
+
+        ss = so.data.decode('utf-8')
+        if ss == '[DONE]':
+            return [None]
+
+        sj = json.loads(ss)  # ChatCompletionChunk
+
+        check.state(sj['object'] == 'chat.completion.chunk')
+
+        ccc = msh.unmarshal(sj, pt.ChatCompletionChunk)
+
+        # FIXME: stop reason
+        if not ccc.choices:
+            return []
+
+        if any(choice.finish_reason for choice in ccc.choices):
+            check.state(all(choice.finish_reason for choice in ccc.choices))
+            return [None]
+
+        return [AiChoicesDeltas([
+            AiChoiceDeltas([build_mc_ai_delta(choice.delta)])
+            for choice in ccc.choices
+        ])]
+
     READ_CHUNK_SIZE: ta.ClassVar[int] = -1
 
     async def invoke(self, request: ChatChoicesStreamRequest) -> ChatChoicesStreamResponse:
-        # check.isinstance(request, ChatRequest)
-
         rh = OpenaiChatRequestHandler(
             request.v,
             *[
@@ -85,7 +102,7 @@ class OpenaiChatChoicesStreamService:
         raw_request = msh.marshal(rh.oai_request())
 
         http_request = http.HttpRequest(
-            'https://api.openai.com/v1/chat/completions',
+            self.URL,
             headers={
                 http.consts.HEADER_CONTENT_TYPE: http.consts.CONTENT_TYPE_JSON,
                 http.consts.HEADER_AUTH: http.consts.format_bearer_auth_header(check.not_none(self._api_key).reveal()),
@@ -93,75 +110,11 @@ class OpenaiChatChoicesStreamService:
             data=json.dumps(raw_request).encode('utf-8'),
         )
 
-        async with UseResources.or_new(request.options) as rs:
-            http_client = await rs.enter_async_context(http.manage_async_client(self._http_client))
-            http_response = await rs.enter_async_context(await http_client.stream_request(http_request))
-
-            if http_response.status != 200:
-                data: ta.Any
-                try:
-                    data = await http_response.stream.readall()
-                except Exception as e:  # noqa
-                    data = e
-                try:
-                    data_obj = json.loads(data.decode())
-                except Exception as e:  # noqa
-                    pass
-                else:
-                    data = data_obj
-                raise OpenaiChatChoicesStreamServiceError(http_response.status, data)
-
-            async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs]:
-                db = DelimitingBuffer([b'\r', b'\n', b'\r\n'])
-                sd = sse.SseDecoder()
-
-                # bs = []
-                # ls = []
-                # sos = []
-
-                while True:
-                    b = await http_response.stream.read1(self.READ_CHUNK_SIZE)
-                    # bs.append(b)
-
-                    for l in db.feed(b):
-                        # ls.append(l)
-
-                        if isinstance(l, DelimitingBuffer.Incomplete):
-                            # FIXME: handle
-                            raise TypeError(l)
-
-                        # FIXME: https://platform.openai.com/docs/guides/function-calling?api-mode=responses#streaming
-                        for so in sd.process_line(l):
-                            # sos.append(so)
-
-                            if isinstance(so, sse.SseEvent) and so.type == b'message':
-                                ss = so.data.decode('utf-8')
-                                if ss == '[DONE]':
-                                    return []
-
-                                sj = json.loads(ss)  # ChatCompletionChunk
-
-                                check.state(sj['object'] == 'chat.completion.chunk')
-
-                                ccc = msh.unmarshal(sj, pt.ChatCompletionChunk)
-
-                                # FIXME: stop reason
-                                if not ccc.choices:
-                                    continue
-
-                                if any(choice.finish_reason for choice in ccc.choices):
-                                    check.state(all(choice.finish_reason for choice in ccc.choices))
-                                    break
-
-                                await sink.emit(AiChoicesDeltas([
-                                    AiChoiceDeltas([build_mc_ai_delta(choice.delta)])
-                                    for choice in ccc.choices
-                                ]))
-
-                    if not b:
-                        return []
-
-            # raw_response = json.loads(check.not_none(http_response.data).decode('utf-8'))
-            # return rh.build_response(raw_response)
-
-            return await new_stream_response(rs, inner)
+        return await BytesHttpStreamResponseBuilder(
+            self._http_client,
+            lambda http_response: SimpleSseLinesHttpStreamResponseHandler(self._process_sse).as_lines().as_bytes(),
+            read_chunk_size=self.READ_CHUNK_SIZE,
+        ).new_stream_response(
+            http_request,
+            request.options,
+        )

ommlds/minichain/http/stream.py

@@ -0,0 +1,195 @@
+"""
+TODO:
+ - better pipeline composition lol
+"""
+import typing as ta
+
+from omlish import check
+from omlish import dataclasses as dc
+from omlish import lang
+from omlish.http import all as http
+from omlish.http import sse
+from omlish.io.buffers import DelimitingBuffer
+
+from ..resources import UseResources
+from ..stream.services import StreamResponse
+from ..stream.services import StreamResponseSink
+from ..stream.services import new_stream_response
+from ..types import Option
+from ..types import Output
+
+
+##
+
+
+@dc.dataclass()
+@dc.extra_class_params(default_repr_fn=lang.opt_repr)
+class HttpStreamResponseError(Exception):
+    response: http.BaseHttpResponse
+
+    data: bytes | None = None
+    data_exception: Exception | None = None
+
+    @classmethod
+    async def from_response(cls, response: http.AsyncStreamHttpResponse) -> 'HttpStreamResponseError':
+        data: bytes | None = None
+        data_exception: Exception | None = None
+
+        try:
+            data = await response.stream.readall()
+        except Exception as de:  # noqa
+            data_exception = de
+
+        return HttpStreamResponseError(
+            response,
+            data=data,
+            data_exception=data_exception,
+        )
+
+
+##
+
+
+class HttpStreamResponseHandler(lang.Abstract):
+    def start(self) -> ta.Sequence[Output]:
+        return ()
+
+    def finish(self) -> ta.Sequence[Output]:
+        return ()
+
+
+##
+
+
+class BytesHttpStreamResponseHandler(HttpStreamResponseHandler, lang.Abstract):
+    def process_bytes(self, data: bytes) -> ta.Iterable:
+        return ()
+
+
+class BytesHttpStreamResponseBuilder:
+    def __init__(
+            self,
+            http_client: http.AsyncHttpClient | None,
+            handling: ta.Callable[[http.AsyncStreamHttpResponse], BytesHttpStreamResponseHandler],
+            *,
+            read_chunk_size: int = -1,
+    ) -> None:
+        super().__init__()
+
+        self._http_client = http_client
+        self._handling = handling
+        self._read_chunk_size = read_chunk_size
+
+    async def new_stream_response(
+            self,
+            http_request: http.HttpRequest,
+            options: ta.Sequence[Option],
+    ) -> StreamResponse:
+        async with UseResources.or_new(options) as rs:
+            http_client = await rs.enter_async_context(http.manage_async_client(self._http_client))
+            http_response = await rs.enter_async_context(await http_client.stream_request(http_request))
+
+            if http_response.status != 200:
+                raise await HttpStreamResponseError.from_response(http_response)
+
+            handler = self._handling(http_response)
+
+            async def inner(sink: StreamResponseSink) -> ta.Sequence | None:
+                while True:
+                    b = await http_response.stream.read1(self._read_chunk_size)
+
+                    for v in handler.process_bytes(b):
+                        if v is None:
+                            break
+
+                        await sink.emit(v)
+
+                    if not b:
+                        break
+
+                return handler.finish()
+
+            return await new_stream_response(
+                rs,
+                inner,
+                handler.start(),
+            )
+
+
+##
+
+
+class LinesHttpStreamResponseHandler(HttpStreamResponseHandler, lang.Abstract):
+    def process_line(self, line: bytes) -> ta.Iterable:
+        return ()
+
+    def as_bytes(self) -> BytesHttpStreamResponseHandler:
+        return LinesBytesHttpStreamResponseHandler(self)
+
+
+class LinesBytesHttpStreamResponseHandler(BytesHttpStreamResponseHandler):
+    def __init__(self, handler: LinesHttpStreamResponseHandler) -> None:
+        super().__init__()
+
+        self._handler = handler
+
+        self._db = DelimitingBuffer([b'\r', b'\n', b'\r\n'])
+
+    def start(self) -> ta.Sequence[Output]:
+        return self._handler.start()
+
+    def process_bytes(self, data: bytes) -> ta.Iterable:
+        for o in self._db.feed(data):
+            if isinstance(o, bytes):
+                yield from self._handler.process_line(o)
+
+            else:
+                raise TypeError(o)
+
+    def finish(self) -> ta.Sequence[Output]:
+        check.state(self._db.is_closed)
+
+        return self._handler.finish()
+
+
+##
+
+
+class SseHttpStreamResponseHandler(HttpStreamResponseHandler, lang.Abstract):
+    def process_sse(self, so: sse.SseDecoderOutput) -> ta.Iterable:
+        return ()
+
+    def as_lines(self) -> LinesHttpStreamResponseHandler:
+        return SseLinesHttpStreamResponseHandler(self)
+
+
+class SseLinesHttpStreamResponseHandler(LinesHttpStreamResponseHandler):
+    def __init__(self, handler: SseHttpStreamResponseHandler) -> None:
+        super().__init__()
+
+        self._handler = handler
+
+        self._sd = sse.SseDecoder()
+
+    def start(self) -> ta.Sequence[Output]:
+        return self._handler.start()
+
+    def process_line(self, line: bytes) -> ta.Iterable:
+        for so in self._sd.process_line(line):
+            yield from self._handler.process_sse(so)
+
+    def finish(self) -> ta.Sequence[Output]:
+        return self._handler.finish()
+
+
+#
+
+
+class SimpleSseLinesHttpStreamResponseHandler(SseHttpStreamResponseHandler):
+    def __init__(self, fn: ta.Callable[[sse.SseDecoderOutput], ta.Iterable]) -> None:
+        super().__init__()
+
+        self._fn = fn
+
+    def process_sse(self, so: sse.SseDecoderOutput) -> ta.Iterable:
+        return self._fn(so)

{ommlds-0.0.0.dev503.dist-info → ommlds-0.0.0.dev505.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ommlds
-Version: 0.0.0.dev503
+Version: 0.0.0.dev505
 Summary: ommlds
 Author: wrmsr
 License-Expression: BSD-3-Clause
@@ -14,14 +14,14 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev503
+Requires-Dist: omlish==0.0.0.dev505
 Provides-Extra: all
-Requires-Dist: omdev==0.0.0.dev503; extra == "all"
+Requires-Dist: omdev==0.0.0.dev505; extra == "all"
 Requires-Dist: llama-cpp-python~=0.3; extra == "all"
 Requires-Dist: mlx~=0.30; sys_platform == "darwin" and extra == "all"
 Requires-Dist: mlx-lm~=0.29; sys_platform == "darwin" and extra == "all"
 Requires-Dist: tiktoken~=0.12; extra == "all"
-Requires-Dist: tinygrad~=0.11; extra == "all"
+Requires-Dist: tinygrad~=0.12; extra == "all"
 Requires-Dist: tokenizers~=0.22; extra == "all"
 Requires-Dist: torch~=2.9; extra == "all"
 Requires-Dist: transformers~=4.57; extra == "all"
@@ -38,13 +38,13 @@ Requires-Dist: mwparserfromhell~=0.7; extra == "all"
 Requires-Dist: wikitextparser~=0.56; extra == "all"
 Requires-Dist: lxml>=5.3; python_version < "3.13" and extra == "all"
 Provides-Extra: omdev
-Requires-Dist: omdev==0.0.0.dev503; extra == "omdev"
+Requires-Dist: omdev==0.0.0.dev505; extra == "omdev"
 Provides-Extra: backends
 Requires-Dist: llama-cpp-python~=0.3; extra == "backends"
 Requires-Dist: mlx~=0.30; sys_platform == "darwin" and extra == "backends"
 Requires-Dist: mlx-lm~=0.29; sys_platform == "darwin" and extra == "backends"
 Requires-Dist: tiktoken~=0.12; extra == "backends"
-Requires-Dist: tinygrad~=0.11; extra == "backends"
+Requires-Dist: tinygrad~=0.12; extra == "backends"
 Requires-Dist: tokenizers~=0.22; extra == "backends"
 Requires-Dist: torch~=2.9; extra == "backends"
 Requires-Dist: transformers~=4.57; extra == "backends"