ommlds 0.0.0.dev426__py3-none-any.whl → 0.0.0.dev485__py3-none-any.whl

ommlds/minichain/backends/impls/groq/stream.py

@@ -0,0 +1,125 @@
+import typing as ta
+
+from omlish import check
+from omlish import marshal as msh
+from omlish import typedvalues as tv
+from omlish.formats import json
+from omlish.http import all as http
+from omlish.http import sse
+from omlish.io.buffers import DelimitingBuffer
+
+from .....backends.groq import protocol as pt
+from ....chat.choices.services import ChatChoicesOutputs
+from ....chat.choices.stream.services import ChatChoicesStreamRequest
+from ....chat.choices.stream.services import ChatChoicesStreamResponse
+from ....chat.choices.stream.services import static_check_is_chat_choices_stream_service
+from ....chat.choices.stream.types import AiChoicesDeltas
+from ....chat.tools.types import Tool
+from ....configs import Config
+from ....resources import UseResources
+from ....standard import ApiKey
+from ....stream.services import StreamResponseSink
+from ....stream.services import new_stream_response
+from .chat import GroqChatChoicesService
+from .names import MODEL_NAMES
+from .protocol import build_gq_request_messages
+from .protocol import build_gq_request_tool
+from .protocol import build_mc_ai_choice_deltas
+
+
+##
+
+
+# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
+#     name='groq',
+#     type='ChatChoicesStreamService',
+# )
+@static_check_is_chat_choices_stream_service
+class GroqChatChoicesStreamService:
+    def __init__(
+            self,
+            *configs: Config,
+            http_client: http.AsyncHttpClient | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._http_client = http_client
+
+        with tv.consume(*configs) as cc:
+            self._model_name = cc.pop(GroqChatChoicesService.DEFAULT_MODEL_NAME)
+            self._api_key = ApiKey.pop_secret(cc, env='GROQ_API_KEY')
+
+    READ_CHUNK_SIZE: ta.ClassVar[int] = -1
+
+    async def invoke(self, request: ChatChoicesStreamRequest) -> ChatChoicesStreamResponse:
+        tools: list[pt.ChatCompletionRequest.Tool] = []
+        with tv.TypedValues(*request.options).consume() as oc:
+            t: Tool
+            for t in oc.pop(Tool, []):
+                tools.append(build_gq_request_tool(t))
+
+        gq_request = pt.ChatCompletionRequest(
+            messages=build_gq_request_messages(request.v),
+            model=MODEL_NAMES.resolve(self._model_name.v),
+            tools=tools or None,
+            stream=True,
+        )
+
+        raw_request = msh.marshal(gq_request)
+
+        http_request = http.HttpRequest(
+            'https://api.groq.com/openai/v1/chat/completions',
+            headers={
+                http.consts.HEADER_CONTENT_TYPE: http.consts.CONTENT_TYPE_JSON,
+                http.consts.HEADER_AUTH: http.consts.format_bearer_auth_header(check.not_none(self._api_key).reveal()),
+            },
+            data=json.dumps(raw_request).encode('utf-8'),
+        )
+
+        async with UseResources.or_new(request.options) as rs:
+            http_client = await rs.enter_async_context(http.manage_async_client(self._http_client))
+            http_response = await rs.enter_async_context(await http_client.stream_request(http_request))
+
+            async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs]:
+                db = DelimitingBuffer([b'\r', b'\n', b'\r\n'])
+                sd = sse.SseDecoder()
+                while True:
+                    b = await http_response.stream.read1(self.READ_CHUNK_SIZE)
+                    for l in db.feed(b):
+                        if isinstance(l, DelimitingBuffer.Incomplete):
+                            # FIXME: handle
+                            return []
+
+                        # FIXME: https://platform.openai.com/docs/guides/function-calling?api-mode=responses#streaming
+                        for so in sd.process_line(l):
+                            if isinstance(so, sse.SseEvent) and so.type == b'message':
+                                ss = so.data.decode('utf-8')
+                                if ss == '[DONE]':
+                                    return []
+
+                                sj = json.loads(ss)  # ChatCompletionChunk
+
+                                check.state(sj['object'] == 'chat.completion.chunk')
+
+                                ccc = msh.unmarshal(sj, pt.ChatCompletionChunk)
+
+                                # FIXME: stop reason
+                                if not ccc.choices:
+                                    continue
+
+                                if any(choice.finish_reason for choice in ccc.choices):
+                                    check.state(all(choice.finish_reason for choice in ccc.choices))
+                                    break
+
+                                await sink.emit(AiChoicesDeltas([
+                                    build_mc_ai_choice_deltas(choice.delta)
+                                    for choice in ccc.choices
+                                ]))
+
+                    if not b:
+                        return []
+
+            # raw_response = json.loads(check.not_none(http_response.data).decode('utf-8'))
+            # return rh.build_response(raw_response)
+
+            return await new_stream_response(rs, inner)

ommlds/minichain/backends/impls/huggingface/repos.py

@@ -3,8 +3,6 @@ TODO:
  - local-only check first
   - cat ~/.cache/.../models/.../refs/main -> c5bfd839cd4cda0e5a39a97e00218d9c56e468af
 """
-import typing as ta
-
 from omlish import lang
 
 from ....models.configs import ModelRepo
@@ -12,10 +10,8 @@ from ....models.repos.resolving import ModelRepoResolver
 from ....models.repos.resolving import ResolvedModelRepo
 
 
-if ta.TYPE_CHECKING:
+with lang.auto_proxy_import(globals()):
     import huggingface_hub as hf
-else:
-    hf = lang.proxy_import('huggingface_hub')
 
 
 ##

ommlds/minichain/backends/impls/llamacpp/chat.py

@@ -2,20 +2,18 @@ import contextlib
 import os.path
 import typing as ta
 
-import llama_cpp as lcc
-
 from omlish import check
 from omlish import lang
 from omlish import typedvalues as tv
 
-from .....backends import llamacpp as lcu
 from ....chat.choices.services import ChatChoicesRequest
 from ....chat.choices.services import ChatChoicesResponse
 from ....chat.choices.services import static_check_is_chat_choices_service
 from ....chat.choices.types import AiChoice
 from ....chat.choices.types import ChatChoicesOptions
 from ....chat.messages import AiMessage
-from ....chat.messages import ToolExecResultMessage
+from ....chat.messages import ToolUseMessage
+from ....chat.messages import ToolUseResultMessage
 from ....chat.tools.types import Tool
 from ....configs import Config
 from ....llms.types import MaxTokens
@@ -26,6 +24,21 @@ from .format import ROLES_MAP
 from .format import get_msg_content
 
 
+with lang.auto_proxy_import(globals()):
+    import llama_cpp as lcc
+
+    from .....backends import llamacpp as lcu
+
+
+##
+
+
+# @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
+#     ['ChatChoicesService'],
+#     'llamacpp',
+# )
+
+
 ##
 
 
@@ -54,7 +67,7 @@ class LlamacppChatChoicesService:
         temperatur=Temperature,
     )
 
-    def invoke(self, request: ChatChoicesRequest) -> ChatChoicesResponse:
+    async def invoke(self, request: ChatChoicesRequest) -> ChatChoicesResponse:
         kwargs: dict = dict(
             # temperature=0,
             max_tokens=1024,
@@ -100,29 +113,34 @@ class LlamacppChatChoicesService:
 
             ims: list = []
             for rm in request.v:
-                if isinstance(rm, ToolExecResultMessage):
+                if isinstance(rm, ToolUseResultMessage):
                     ims.append(dict(
                         role='tool',
-                        **(dict(id=rm.id) if rm.id is not None else {}),
-                        name=rm.name,
-                        content=check.isinstance(rm.c, str),
+                        **(dict(id=rm.tur.id) if rm.tur.id is not None else {}),
+                        name=rm.tur.name,
+                        content=check.isinstance(rm.tur.c, str),
                     ))
+
                 elif isinstance(rm, AiMessage):
-                    tcs: list[dict] = []
-                    for ter in rm.tool_exec_requests or []:
-                        tcs.append(dict(
-                            id=check.not_none(ter.id),
-                            type='function',
-                            function=dict(
-                                name=ter.name,
-                                arguments=check.isinstance(ter.raw_args, str),
-                            ),
-                        ))
                     ims.append(dict(
                         role=ROLES_MAP[type(rm)],
                         **(dict(content=mc) if (mc := get_msg_content(rm)) is not None else {}),
-                        **(dict(tool_calls=tcs) if tcs else {}),
                     ))
+
+                elif isinstance(rm, ToolUseMessage):
+                    ims.append(dict(
+                        role=ROLES_MAP[type(rm)],
+                        content='',
+                        tool_calls=[dict(
+                            id=check.not_none(rm.tu.id),
+                            type='function',
+                            function=dict(
+                                name=rm.tu.name,
+                                arguments=check.isinstance(rm.tu.raw_args, str),
+                            ),
+                        )],
+                    ))
+
                 else:
                     ims.append(dict(
                         role=ROLES_MAP[type(rm)],
@@ -135,8 +153,8 @@ class LlamacppChatChoicesService:
             )
 
             out: list[AiChoice] = []
-            for c in output['choices']:  # type: ignore
+            for c in ta.cast(ta.Any, output)['choices']:
                 m = c['message']
-                out.append(AiChoice(AiMessage(m['content'])))
+                out.append(AiChoice([AiMessage(m['content'])]))
 
             return ChatChoicesResponse(out)

ommlds/minichain/backends/impls/llamacpp/completion.py

@@ -2,11 +2,9 @@ import contextlib
 import os.path
 import typing as ta
 
-import llama_cpp as lcc
-
+from omlish import lang
 from omlish import typedvalues as tv
 
-from .....backends import llamacpp as lcu
 from ....completion import CompletionOption
 from ....completion import CompletionRequest
 from ....completion import CompletionResponse
@@ -18,6 +16,12 @@ from ....llms.types import Temperature
 from ....models.configs import ModelPath
 
 
+with lang.auto_proxy_import(globals()):
+    import llama_cpp as lcc
+
+    from .....backends import llamacpp as lcu
+
+
 ##
 
 
@@ -51,7 +55,7 @@ class LlamacppCompletionService:
         temperatur=Temperature,
     )
 
-    def invoke(self, request: CompletionRequest) -> CompletionResponse:
+    async def invoke(self, request: CompletionRequest) -> CompletionResponse:
         kwargs: dict = dict(
             # temperature=0,
             max_tokens=1024,
@@ -74,4 +78,4 @@ class LlamacppCompletionService:
                 **kwargs,
             )
 
-            return CompletionResponse(output['choices'][0]['text'])  # type: ignore
+            return CompletionResponse(ta.cast(ta.Any, output)['choices'][0]['text'])

ommlds/minichain/backends/impls/llamacpp/format.py

@@ -5,7 +5,8 @@ from omlish import check
 from ....chat.messages import AiMessage
 from ....chat.messages import Message
 from ....chat.messages import SystemMessage
-from ....chat.messages import ToolExecResultMessage
+from ....chat.messages import ToolUseMessage
+from ....chat.messages import ToolUseResultMessage
 from ....chat.messages import UserMessage
 
 
@@ -16,7 +17,8 @@ ROLES_MAP: ta.Mapping[type[Message], str] = {
     SystemMessage: 'system',
     UserMessage: 'user',
     AiMessage: 'assistant',
-    ToolExecResultMessage: 'tool',
+    ToolUseMessage: 'assistant',
+    ToolUseResultMessage: 'tool',
 }
 
 

ommlds/minichain/backends/impls/llamacpp/stream.py

@@ -2,29 +2,42 @@ import contextlib
 import threading
 import typing as ta  # noqa
 
-import llama_cpp as lcc
-
 from omlish import check
 from omlish import lang
 from omlish import typedvalues as tv
 
-from .....backends import llamacpp as lcu
 from ....chat.choices.services import ChatChoicesOutputs
-from ....chat.stream.services import ChatChoicesStreamRequest
-from ....chat.stream.services import ChatChoicesStreamResponse
-from ....chat.stream.services import static_check_is_chat_choices_stream_service
-from ....chat.stream.types import AiChoiceDelta
-from ....chat.stream.types import AiChoiceDeltas
-from ....chat.stream.types import AiMessageDelta
+from ....chat.choices.stream.services import ChatChoicesStreamRequest
+from ....chat.choices.stream.services import ChatChoicesStreamResponse
+from ....chat.choices.stream.services import static_check_is_chat_choices_stream_service
+from ....chat.choices.stream.types import AiChoiceDeltas
+from ....chat.choices.stream.types import AiChoicesDeltas
+from ....chat.stream.types import ContentAiDelta
 from ....configs import Config
 from ....models.configs import ModelPath
 from ....resources import UseResources
+from ....stream.services import StreamResponseSink
 from ....stream.services import new_stream_response
 from .chat import LlamacppChatChoicesService
 from .format import ROLES_MAP
 from .format import get_msg_content
 
 
+with lang.auto_proxy_import(globals()):
+    import llama_cpp as lcc
+
+    from .....backends import llamacpp as lcu
+
+
+##
+
+
+# @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
+#     ['ChatChoicesStreamService'],
+#     'llamacpp',
+# )
+
+
 ##
 
 
@@ -49,10 +62,10 @@ class LlamacppChatChoicesStreamService(lang.ExitStacked):
             verbose=False,
         )))
 
-    def invoke(self, request: ChatChoicesStreamRequest) -> ChatChoicesStreamResponse:
+    async def invoke(self, request: ChatChoicesStreamRequest) -> ChatChoicesStreamResponse:
         lcu.install_logging_hook()
 
-        with UseResources.or_new(request.options) as rs:
+        async with UseResources.or_new(request.options) as rs:
             rs.enter_context(self._lock)
 
             model: ta.Any = self._load_model()  # FIXME: the types are awful lol
@@ -74,19 +87,26 @@ class LlamacppChatChoicesStreamService(lang.ExitStacked):
 
             rs.enter_context(lang.defer(close_output))
 
-            def yield_choices() -> ta.Generator[AiChoiceDeltas, None, ta.Sequence[ChatChoicesOutputs] | None]:
+            async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs] | None:
+                last_role: ta.Any = None
+
                 for chunk in output:
                     check.state(chunk['object'] == 'chat.completion.chunk')
-                    l: list[AiChoiceDelta] = []
-                    for choice in chunk['choices']:
-                        # FIXME: check role is assistant
-                        # FIXME: stop reason
-                        if not (delta := choice.get('delta', {})):
-                            continue
-                        if not (content := delta.get('content', '')):
-                            continue
-                        l.append(AiChoiceDelta(AiMessageDelta(content)))
-                    yield l
+
+                    choice = check.single(chunk['choices'])
+
+                    if not (delta := choice.get('delta', {})):
+                        continue
+
+                    # FIXME: check role is assistant
+                    if (role := delta.get('role')) != last_role:
+                        last_role = role
+
+                    # FIXME: stop reason
+
+                    if (content := delta.get('content', '')):
+                        await sink.emit(AiChoicesDeltas([AiChoiceDeltas([ContentAiDelta(content)])]))
+
                 return None
 
-            return new_stream_response(rs, yield_choices())
+            return await new_stream_response(rs, inner)

ommlds/minichain/backends/impls/mistral.py

@@ -21,13 +21,18 @@ from ...chat.messages import UserMessage
 ##
 
 
+# TODO: generalize lol
+class TooManyRequestsMistralError(Exception):
+    pass
+
+
 # @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
 #     name='mistral',
 #     type='ChatChoicesService',
 # )
 @static_check_is_chat_choices_service
 class MistralChatChoicesService:
-    model: ta.ClassVar[str] = 'mistral-large-latest'
+    model: ta.ClassVar[str] = 'mistral-medium-2508'
 
     ROLES_MAP: ta.ClassVar[ta.Mapping[type[Message], str]] = {
         SystemMessage: 'system',
@@ -35,10 +40,16 @@ class MistralChatChoicesService:
         AiMessage: 'assistant',
     }
 
-    def __init__(self, *, api_key: str | None = None) -> None:
+    def __init__(
+            self,
+            *,
+            api_key: str | None = None,
+            http_client: http.AsyncHttpClient | None = None,
+    ) -> None:
         super().__init__()
 
         self._api_key = api_key
+        self._http_client = http_client
 
     def _get_msg_content(self, m: Message) -> str | None:
         if isinstance(m, AiMessage):
@@ -50,7 +61,7 @@ class MistralChatChoicesService:
         else:
             raise TypeError(m)
 
-    def invoke(
+    async def invoke(
             self,
             request: ChatChoicesRequest,
     ) -> ChatChoicesResponse:
@@ -68,7 +79,7 @@ class MistralChatChoicesService:
             ],
         }
 
-        resp = http.request(
+        resp = await http.async_request(
             'https://api.mistral.ai/v1/chat/completions',
             method='POST',
             data=json.dumps_compact(req_dct).encode('utf-8'),
@@ -77,11 +88,15 @@ class MistralChatChoicesService:
                 'Accept': 'application/json',
                 'Authorization': f'Bearer {key}',
             },
+            client=self._http_client,
         )
 
+        if resp.status == 429:
+            raise TooManyRequestsMistralError
+
         resp_dct = json.loads(check.not_none(resp.data).decode('utf-8'))
 
         return ChatChoicesResponse([
-            AiChoice(AiMessage(c['message']['content']))
+            AiChoice([AiMessage(c['message']['content'])])
             for c in resp_dct['choices']
         ])