ommlds 0.0.0.dev466__py3-none-any.whl → 0.0.0.dev512__py3-none-any.whl

ommlds/minichain/backends/impls/llamacpp/chat.py

@@ -2,13 +2,10 @@ import contextlib
 import os.path
 import typing as ta
 
-import llama_cpp as lcc
-
 from omlish import check
 from omlish import lang
 from omlish import typedvalues as tv
 
-from .....backends import llamacpp as lcu
 from ....chat.choices.services import ChatChoicesRequest
 from ....chat.choices.services import ChatChoicesResponse
 from ....chat.choices.services import static_check_is_chat_choices_service
@@ -27,6 +24,21 @@ from .format import ROLES_MAP
 from .format import get_msg_content
 
 
+with lang.auto_proxy_import(globals()):
+    import llama_cpp as lcc
+
+    from .....backends import llamacpp as lcu
+
+
+##
+
+
+# @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
+#     ['ChatChoicesService'],
+#     'llamacpp',
+# )
+
+
 ##
 
 

ommlds/minichain/backends/impls/llamacpp/completion.py

@@ -2,11 +2,9 @@ import contextlib
 import os.path
 import typing as ta
 
-import llama_cpp as lcc
-
+from omlish import lang
 from omlish import typedvalues as tv
 
-from .....backends import llamacpp as lcu
 from ....completion import CompletionOption
 from ....completion import CompletionRequest
 from ....completion import CompletionResponse
@@ -18,6 +16,12 @@ from ....llms.types import Temperature
 from ....models.configs import ModelPath
 
 
+with lang.auto_proxy_import(globals()):
+    import llama_cpp as lcc
+
+    from .....backends import llamacpp as lcu
+
+
 ##
 
 

ommlds/minichain/backends/impls/llamacpp/stream.py

@@ -2,20 +2,17 @@ import contextlib
 import threading
 import typing as ta  # noqa
 
-import llama_cpp as lcc
-
 from omlish import check
 from omlish import lang
 from omlish import typedvalues as tv
 
-from .....backends import llamacpp as lcu
 from ....chat.choices.services import ChatChoicesOutputs
-from ....chat.stream.services import ChatChoicesStreamRequest
-from ....chat.stream.services import ChatChoicesStreamResponse
-from ....chat.stream.services import static_check_is_chat_choices_stream_service
-from ....chat.stream.types import AiChoiceDeltas
-from ....chat.stream.types import AiChoicesDeltas
-from ....chat.stream.types import ContentAiChoiceDelta
+from ....chat.choices.stream.services import ChatChoicesStreamRequest
+from ....chat.choices.stream.services import ChatChoicesStreamResponse
+from ....chat.choices.stream.services import static_check_is_chat_choices_stream_service
+from ....chat.choices.stream.types import AiChoiceDeltas
+from ....chat.choices.stream.types import AiChoicesDeltas
+from ....chat.stream.types import ContentAiDelta
 from ....configs import Config
 from ....models.configs import ModelPath
 from ....resources import UseResources
@@ -26,6 +23,21 @@ from .format import ROLES_MAP
 from .format import get_msg_content
 
 
+with lang.auto_proxy_import(globals()):
+    import llama_cpp as lcc
+
+    from .....backends import llamacpp as lcu
+
+
+##
+
+
+# @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
+#     ['ChatChoicesStreamService'],
+#     'llamacpp',
+# )
+
+
 ##
 
 
@@ -76,18 +88,25 @@ class LlamacppChatChoicesStreamService(lang.ExitStacked):
             rs.enter_context(lang.defer(close_output))
 
             async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs] | None:
+                last_role: ta.Any = None
+
                 for chunk in output:
                     check.state(chunk['object'] == 'chat.completion.chunk')
-                    l: list[AiChoiceDeltas] = []
-                    for choice in chunk['choices']:
-                        # FIXME: check role is assistant
-                        # FIXME: stop reason
-                        if not (delta := choice.get('delta', {})):
-                            continue
-                        if not (content := delta.get('content', '')):
-                            continue
-                        l.append(AiChoiceDeltas([ContentAiChoiceDelta(content)]))
-                    await sink.emit(AiChoicesDeltas(l))
+
+                    choice = check.single(chunk['choices'])
+
+                    if not (delta := choice.get('delta', {})):
+                        continue
+
+                    # FIXME: check role is assistant
+                    if (role := delta.get('role')) != last_role:
+                        last_role = role
+
+                    # FIXME: stop reason
+
+                    if (content := delta.get('content', '')):
+                        await sink.emit(AiChoicesDeltas([AiChoiceDeltas([ContentAiDelta(content)])]))
+
                 return None
 
             return await new_stream_response(rs, inner)

ommlds/minichain/backends/impls/mistral.py

@@ -40,10 +40,16 @@ class MistralChatChoicesService:
         AiMessage: 'assistant',
     }
 
-    def __init__(self, *, api_key: str | None = None) -> None:
+    def __init__(
+            self,
+            *,
+            api_key: str | None = None,
+            http_client: http.AsyncHttpClient | None = None,
+    ) -> None:
         super().__init__()
 
         self._api_key = api_key
+        self._http_client = http_client
 
     def _get_msg_content(self, m: Message) -> str | None:
         if isinstance(m, AiMessage):
@@ -73,7 +79,7 @@ class MistralChatChoicesService:
             ],
         }
 
-        resp = http.request(
+        resp = await http.async_request(
             'https://api.mistral.ai/v1/chat/completions',
             method='POST',
             data=json.dumps_compact(req_dct).encode('utf-8'),
@@ -82,6 +88,7 @@ class MistralChatChoicesService:
                 'Accept': 'application/json',
                 'Authorization': f'Bearer {key}',
             },
+            client=self._http_client,
         )
 
         if resp.status == 429:

ommlds/minichain/backends/impls/mlx/chat.py

@@ -1,32 +1,46 @@
+import contextlib
 import typing as ta
 
 from omlish import check
 from omlish import lang
 from omlish import typedvalues as tv
 
-from .....backends import mlx as mlxu
+from ....chat.choices.services import ChatChoicesOutputs
 from ....chat.choices.services import ChatChoicesRequest
 from ....chat.choices.services import ChatChoicesResponse
 from ....chat.choices.services import static_check_is_chat_choices_service
+from ....chat.choices.stream.services import ChatChoicesStreamRequest
+from ....chat.choices.stream.services import ChatChoicesStreamResponse
+from ....chat.choices.stream.services import static_check_is_chat_choices_stream_service
+from ....chat.choices.stream.types import AiChoiceDeltas
+from ....chat.choices.stream.types import AiChoicesDeltas
 from ....chat.choices.types import AiChoice
 from ....chat.choices.types import ChatChoicesOptions
 from ....chat.messages import AiMessage
 from ....chat.messages import Message
 from ....chat.messages import SystemMessage
 from ....chat.messages import UserMessage
+from ....chat.stream.types import ContentAiDelta
 from ....configs import Config
 from ....llms.types import MaxTokens
 from ....models.configs import ModelPath
 from ....models.configs import ModelRepo
 from ....models.configs import ModelSpecifier
+from ....resources import UseResources
 from ....standard import DefaultOptions
+from ....stream.services import StreamResponseSink
+from ....stream.services import new_stream_response
+
+
+with lang.auto_proxy_import(globals()):
+    from .....backends import mlx as mlxu
 
 
 ##
 
 
 # @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
-#     ['ChatChoicesService'],
+#     ['ChatChoicesService', 'ChatChoicesStreamService'],
 #     'mlx',
 # )
 
@@ -34,12 +48,7 @@ from ....standard import DefaultOptions
 ##
 
 
-# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
-#     name='mlx',
-#     type='ChatChoicesService',
-# )
-@static_check_is_chat_choices_service
-class MlxChatChoicesService(lang.ExitStacked):
+class BaseMlxChatChoicesService(lang.ExitStacked):
     DEFAULT_MODEL: ta.ClassVar[ModelSpecifier] = (
         # 'mlx-community/DeepSeek-Coder-V2-Lite-Instruct-8bit'
         # 'mlx-community/Llama-3.3-70B-Instruct-4bit'
@@ -52,8 +61,8 @@ class MlxChatChoicesService(lang.ExitStacked):
         # 'mlx-community/Qwen2.5-0.5B-4bit'
         # 'mlx-community/Qwen2.5-32B-Instruct-8bit'
         # 'mlx-community/Qwen2.5-Coder-32B-Instruct-8bit'
-        # 'mlx-community/mamba-2.8b-hf-f16'
         # 'mlx-community/Qwen3-30B-A3B-6bit'
+        # 'mlx-community/mamba-2.8b-hf-f16'
     )
 
     def __init__(self, *configs: Config) -> None:
@@ -70,17 +79,14 @@ class MlxChatChoicesService(lang.ExitStacked):
     }
 
     def _get_msg_content(self, m: Message) -> str | None:
-        if isinstance(m, AiMessage):
-            return check.isinstance(m.c, str)
-
-        elif isinstance(m, (SystemMessage, UserMessage)):
+        if isinstance(m, (AiMessage, SystemMessage, UserMessage)):
             return check.isinstance(m.c, str)
 
         else:
             raise TypeError(m)
 
     @lang.cached_function(transient=True)
-    def _load_model(self) -> mlxu.LoadedModel:
+    def _load_model(self) -> 'mlxu.LoadedModel':
         # FIXME: walk state, find all mx.arrays, dealloc/set to empty
         check.not_none(self._exit_stack)
 
@@ -96,10 +102,9 @@ class MlxChatChoicesService(lang.ExitStacked):
         max_tokens=MaxTokens,
     )
 
-    async def invoke(self, request: ChatChoicesRequest) -> ChatChoicesResponse:
-        loaded_model = self._load_model()
-
-        tokenizer = loaded_model.tokenization.tokenizer
+    @lang.cached_function(transient=True)
+    def _get_tokenizer(self) -> 'mlxu.tokenization.Tokenizer':
+        tokenizer = self._load_model().tokenization.tokenizer
 
         if not (
                 hasattr(tokenizer, 'apply_chat_template') and
@@ -107,26 +112,44 @@ class MlxChatChoicesService(lang.ExitStacked):
         ):
             raise RuntimeError(tokenizer)
 
-        prompt = tokenizer.apply_chat_template(
+        return tokenizer
+
+    def _build_prompt(self, messages: ta.Sequence[Message]) -> str:
+        return check.isinstance(self._get_tokenizer().apply_chat_template(
             [  # type: ignore[arg-type]
                 dict(
                     role=self.ROLES_MAP[type(m)],
                     content=self._get_msg_content(m),
                 )
-                for m in request.v
+                for m in messages
             ],
             tokenize=False,
             add_generation_prompt=True,
-        )
+        ), str)
+
+    def _build_kwargs(self, oc: tv.TypedValuesConsumer) -> dict[str, ta.Any]:
+        kwargs: dict[str, ta.Any] = {}
+        kwargs.update(oc.pop_scalar_kwargs(**self._OPTION_KWARG_NAMES_MAP))
+        return kwargs
 
-        kwargs = dict()
+
+# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
+#     name='mlx',
+#     type='ChatChoicesService',
+# )
+@static_check_is_chat_choices_service
+class MlxChatChoicesService(BaseMlxChatChoicesService):
+    async def invoke(self, request: ChatChoicesRequest) -> ChatChoicesResponse:
+        loaded_model = self._load_model()
+
+        prompt = self._build_prompt(request.v)
 
         with tv.consume(
                 *self._default_options,
                 *request.options,
                 override=True,
         ) as oc:
-            kwargs.update(oc.pop_scalar_kwargs(**self._OPTION_KWARG_NAMES_MAP))
+            kwargs = self._build_kwargs(oc)
 
         response = mlxu.generate(
             loaded_model.model,
@@ -139,3 +162,57 @@ class MlxChatChoicesService(lang.ExitStacked):
         return ChatChoicesResponse([
             AiChoice([AiMessage(response)])  # noqa
         ])
+
+
+# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
+#     name='mlx',
+#     type='ChatChoicesStreamService',
+# )
+@static_check_is_chat_choices_stream_service
+class MlxChatChoicesStreamService(BaseMlxChatChoicesService):
+    def __init__(self, *configs: Config) -> None:
+        super().__init__()
+
+        with tv.consume(*configs) as cc:
+            self._model = cc.pop(MlxChatChoicesService.DEFAULT_MODEL)
+            self._default_options: tv.TypedValues = DefaultOptions.pop(cc)
+
+    READ_CHUNK_SIZE = 64 * 1024
+
+    async def invoke(
+            self,
+            request: ChatChoicesStreamRequest,
+            *,
+            max_tokens: int = 4096,  # FIXME: ChatOption
+    ) -> ChatChoicesStreamResponse:
+        loaded_model = self._load_model()
+
+        prompt = self._build_prompt(request.v)
+
+        with tv.consume(
+                *self._default_options,
+                *request.options,
+                override=True,
+        ) as oc:
+            oc.pop(UseResources, None)
+            kwargs = self._build_kwargs(oc)
+
+        async with UseResources.or_new(request.options) as rs:
+            gen: ta.Iterator[mlxu.GenerationOutput] = rs.enter_context(contextlib.closing(mlxu.stream_generate(
+                loaded_model.model,
+                loaded_model.tokenization,
+                check.isinstance(prompt, str),
+                mlxu.GenerationParams(**kwargs),
+                # verbose=True,
+            )))
+
+            async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs]:
+                for go in gen:
+                    if go.text:
+                        await sink.emit(AiChoicesDeltas([AiChoiceDeltas([
+                            ContentAiDelta(go.text),
+                        ])]))
+
+                return []
+
+            return await new_stream_response(rs, inner)

ommlds/minichain/backends/impls/ollama/chat.py

@@ -0,0 +1,193 @@
+"""
+dolphin3:latest           d5ab9ae8e1f2    4.9 GB    11 months ago (no tools)
+
+functiongemma:270m        7c19b650567a    300 MB    5 minutes ago
+gemma3:27b                a418f5838eaf    17 GB     7 weeks ago (no tools)
+gemma3:4b                 a2af6cc3eb7f    3.3 GB    7 weeks ago (no tools)
+
+llama3.2:1b               baf6a787fdff    1.3 GB    13 months ago (too stupid for tools)
+llama3.2:latest           a80c4f17acd5    2.0 GB    13 months ago
+
+devstral-small-2:24b      24277f07f62d    15 GB     15 hours ago
+ministral-3:14b           4760c35aeb9d    9.1 GB    11 hours ago
+mistral:latest            6577803aa9a0    4.4 GB    3 seconds ago
+
+nemotron-3-nano:30b       b725f1117407    24 GB     15 hours ago
+
+olmo-3.1:32b-instruct     a16b6a5be6cf    19 GB     11 hours ago (no tools)
+olmo-3.1:32b-think        223d4ec84d91    19 GB     11 hours ago (no tools)
+
+phi4-mini:latest          78fad5d182a7    2.5 GB    8 months ago (no tools)
+
+qwen3-coder:30b           06c1097efce0    18 GB     11 hours ago
+qwen3-next:80b            b2ebb986e4e9    50 GB     11 hours ago
+qwen3:30b                 ad815644918f    18 GB     11 hours ago
+qwen3:32b                 030ee887880f    20 GB     11 hours ago
+"""
+import typing as ta
+
+from omlish import check
+from omlish import lang
+from omlish import marshal as msh
+from omlish import typedvalues as tv
+from omlish.formats import json
+from omlish.http import all as http
+from omlish.io.buffers import DelimitingBuffer
+
+from .....backends.ollama import protocol as pt
+from ....chat.choices.services import ChatChoicesOutputs
+from ....chat.choices.services import ChatChoicesRequest
+from ....chat.choices.services import ChatChoicesResponse
+from ....chat.choices.services import static_check_is_chat_choices_service
+from ....chat.choices.stream.services import ChatChoicesStreamRequest
+from ....chat.choices.stream.services import ChatChoicesStreamResponse
+from ....chat.choices.stream.services import static_check_is_chat_choices_stream_service
+from ....chat.choices.stream.types import AiChoicesDeltas
+from ....chat.tools.types import Tool
+from ....models.configs import ModelName
+from ....resources import UseResources
+from ....standard import ApiUrl
+from ....stream.services import StreamResponseSink
+from ....stream.services import new_stream_response
+from .protocol import build_mc_ai_choice_deltas
+from .protocol import build_mc_choices_response
+from .protocol import build_ol_request_messages
+from .protocol import build_ol_request_tool
+
+
+##
+
+
+# @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
+#     [
+#         'ChatChoicesService',
+#         'ChatChoicesStreamService',
+#     ],
+#     'ollama',
+# )
+
+
+##
+
+
+class BaseOllamaChatChoicesService(lang.Abstract):
+    DEFAULT_API_URL: ta.ClassVar[ApiUrl] = ApiUrl('http://localhost:11434/api')
+    DEFAULT_MODEL_NAME: ta.ClassVar[ModelName] = ModelName('llama3.2')
+
+    def __init__(
+            self,
+            *configs: ApiUrl | ModelName,
+            http_client: http.AsyncHttpClient | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._http_client = http_client
+
+        with tv.consume(*configs) as cc:
+            self._api_url = cc.pop(self.DEFAULT_API_URL)
+            self._model_name = cc.pop(self.DEFAULT_MODEL_NAME)
+
+
+##
+
+
+# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
+#     name='ollama',
+#     type='ChatChoicesService',
+# )
+@static_check_is_chat_choices_service
+class OllamaChatChoicesService(BaseOllamaChatChoicesService):
+    async def invoke(
+            self,
+            request: ChatChoicesRequest,
+    ) -> ChatChoicesResponse:
+        messages = build_ol_request_messages(request.v)
+
+        tools: list[pt.Tool] = []
+        with tv.TypedValues(*request.options).consume() as oc:
+            t: Tool
+            for t in oc.pop(Tool, []):
+                tools.append(build_ol_request_tool(t))
+
+        a_req = pt.ChatRequest(
+            model=self._model_name.v,
+            messages=messages,
+            tools=tools or None,
+            stream=False,
+        )
+
+        raw_request = msh.marshal(a_req)
+
+        async with http.manage_async_client(self._http_client) as http_client:
+            raw_response = await http_client.request(http.HttpRequest(
+                self._api_url.v.removesuffix('/') + '/chat',
+                data=json.dumps(raw_request).encode('utf-8'),
+            ))
+
+        json_response = json.loads(check.not_none(raw_response.data).decode('utf-8'))
+
+        resp = msh.unmarshal(json_response, pt.ChatResponse)
+
+        return build_mc_choices_response(resp)
+
+
+##
+
+
+# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
+#     name='ollama',
+#     type='ChatChoicesStreamService',
+# )
+@static_check_is_chat_choices_stream_service
+class OllamaChatChoicesStreamService(BaseOllamaChatChoicesService):
+    READ_CHUNK_SIZE: ta.ClassVar[int] = -1
+
+    async def invoke(
+            self,
+            request: ChatChoicesStreamRequest,
+    ) -> ChatChoicesStreamResponse:
+        messages = build_ol_request_messages(request.v)
+
+        tools: list[pt.Tool] = []
+        with tv.TypedValues(*request.options).consume() as oc:
+            t: Tool
+            for t in oc.pop(Tool, []):
+                tools.append(build_ol_request_tool(t))
+
+        a_req = pt.ChatRequest(
+            model=self._model_name.v,
+            messages=messages,
+            tools=tools or None,
+            stream=True,
+        )
+
+        raw_request = msh.marshal(a_req)
+
+        http_request = http.HttpRequest(
+            self._api_url.v.removesuffix('/') + '/chat',
+            data=json.dumps(raw_request).encode('utf-8'),
+        )
+
+        async with UseResources.or_new(request.options) as rs:
+            http_client = await rs.enter_async_context(http.manage_async_client(self._http_client))
+            http_response = await rs.enter_async_context(await http_client.stream_request(http_request))
+
+            async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs] | None:
+                db = DelimitingBuffer([b'\r', b'\n', b'\r\n'])
+                while True:
+                    b = await http_response.stream.read1(self.READ_CHUNK_SIZE)
+                    for l in db.feed(b):
+                        if isinstance(l, DelimitingBuffer.Incomplete):
+                            # FIXME: handle
+                            raise TypeError(l)
+
+                        lj = json.loads(l.decode('utf-8'))
+                        lp: pt.ChatResponse = msh.unmarshal(lj, pt.ChatResponse)
+
+                        if (ds := build_mc_ai_choice_deltas(lp)).deltas:
+                            await sink.emit(AiChoicesDeltas([ds]))
+
+                    if not b:
+                        return []
+
+            return await new_stream_response(rs, inner)