ommlds 0.0.0.dev467__py3-none-any.whl → 0.0.0.dev468__py3-none-any.whl

ommlds/.omlish-manifests.json

@@ -170,6 +170,21 @@
     "attr": null,
     "file": "ommlds/minichain/backends/impls/llamacpp/chat.py",
     "line": 33,
+    "value": {
+      "!.minichain.backends.strings.manifests.BackendStringsManifest": {
+        "service_cls_names": [
+          "ChatChoicesService"
+        ],
+        "backend_name": "llamacpp",
+        "model_names": null
+      }
+    }
+  },
+  {
+    "module": ".minichain.backends.impls.llamacpp.chat",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/llamacpp/chat.py",
+    "line": 42,
     "value": {
       "!.minichain.registries.manifests.RegistryManifest": {
         "module": "ommlds.minichain.backends.impls.llamacpp.chat",
@@ -200,6 +215,21 @@
     "attr": null,
     "file": "ommlds/minichain/backends/impls/llamacpp/stream.py",
     "line": 32,
+    "value": {
+      "!.minichain.backends.strings.manifests.BackendStringsManifest": {
+        "service_cls_names": [
+          "ChatChoicesStreamService"
+        ],
+        "backend_name": "llamacpp",
+        "model_names": null
+      }
+    }
+  },
+  {
+    "module": ".minichain.backends.impls.llamacpp.stream",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/llamacpp/stream.py",
+    "line": 41,
     "value": {
       "!.minichain.registries.manifests.RegistryManifest": {
         "module": "ommlds.minichain.backends.impls.llamacpp.stream",
@@ -271,6 +301,52 @@
       }
     }
   },
+  {
+    "module": ".minichain.backends.impls.ollama.chat",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/ollama/chat.py",
+    "line": 38,
+    "value": {
+      "!.minichain.backends.strings.manifests.BackendStringsManifest": {
+        "service_cls_names": [
+          "ChatChoicesService",
+          "ChatChoicesStreamService"
+        ],
+        "backend_name": "ollama",
+        "model_names": null
+      }
+    }
+  },
+  {
+    "module": ".minichain.backends.impls.ollama.chat",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/ollama/chat.py",
+    "line": 93,
+    "value": {
+      "!.minichain.registries.manifests.RegistryManifest": {
+        "module": "ommlds.minichain.backends.impls.ollama.chat",
+        "attr": "OllamaChatChoicesService",
+        "name": "ollama",
+        "aliases": null,
+        "type": "ChatChoicesService"
+      }
+    }
+  },
+  {
+    "module": ".minichain.backends.impls.ollama.chat",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/ollama/chat.py",
+    "line": 139,
+    "value": {
+      "!.minichain.registries.manifests.RegistryManifest": {
+        "module": "ommlds.minichain.backends.impls.ollama.chat",
+        "attr": "OllamaChatChoicesStreamService",
+        "name": "ollama",
+        "aliases": null,
+        "type": "ChatChoicesStreamService"
+      }
+    }
+  },
   {
     "module": ".minichain.backends.impls.openai.chat",
     "attr": null,
@@ -450,7 +526,23 @@
     "module": ".minichain.backends.impls.transformers.transformers",
     "attr": null,
     "file": "ommlds/minichain/backends/impls/transformers/transformers.py",
-    "line": 43,
+    "line": 46,
+    "value": {
+      "!.minichain.backends.strings.manifests.BackendStringsManifest": {
+        "service_cls_names": [
+          "ChatChoicesService",
+          "ChatChoicesStreamService"
+        ],
+        "backend_name": "transformers",
+        "model_names": null
+      }
+    }
+  },
+  {
+    "module": ".minichain.backends.impls.transformers.transformers",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/transformers/transformers.py",
+    "line": 62,
     "value": {
       "!.minichain.registries.manifests.RegistryManifest": {
         "module": "ommlds.minichain.backends.impls.transformers.transformers",
@@ -467,7 +559,7 @@
     "module": ".minichain.backends.impls.transformers.transformers",
     "attr": null,
     "file": "ommlds/minichain/backends/impls/transformers/transformers.py",
-    "line": 131,
+    "line": 189,
     "value": {
       "!.minichain.registries.manifests.RegistryManifest": {
         "module": "ommlds.minichain.backends.impls.transformers.transformers",
@@ -480,6 +572,21 @@
       }
     }
   },
+  {
+    "module": ".minichain.backends.impls.transformers.transformers",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/transformers/transformers.py",
+    "line": 219,
+    "value": {
+      "!.minichain.registries.manifests.RegistryManifest": {
+        "module": "ommlds.minichain.backends.impls.transformers.transformers",
+        "attr": "TransformersChatChoicesStreamService",
+        "name": "transformers",
+        "aliases": null,
+        "type": "ChatChoicesStreamService"
+      }
+    }
+  },
   {
     "module": ".minichain.chat.choices.services",
     "attr": null,

ommlds/__about__.py

@@ -37,8 +37,8 @@ class Project(ProjectBase):
         ],
 
         'huggingface': [
-            'huggingface-hub ~= 0.35',
-            'datasets ~= 4.2',
+            'huggingface-hub ~= 0.36',
+            'datasets ~= 4.3',
         ],
 
         'numpy': [

ommlds/backends/ollama/protocol.py

@@ -0,0 +1,170 @@
+"""
+https://docs.ollama.com/api
+"""
+import typing as ta
+
+from omlish import dataclasses as dc
+from omlish import lang
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class Options:
+    # loading
+    numa: bool | None = None
+    num_ctx: int | None = None
+    num_batch: int | None = None
+    num_gpu: int | None = None
+    main_gpu: int | None = None
+    low_vram: bool | None = None
+    f16_kv: bool | None = None
+    logits_all: bool | None = None
+    vocab_only: bool | None = None
+    use_mmap: bool | None = None
+    use_mlock: bool | None = None
+    embedding_only: bool | None = None
+    num_thread: int | None = None
+
+    # querying
+    num_keep: int | None = None
+    seed: int | None = None
+    num_predict: int | None = None
+    top_k: int | None = None
+    top_p: float | None = None
+    tfs_z: float | None = None
+    typical_p: float | None = None
+    repeat_last_n: int | None = None
+    temperature: float | None = None
+    repeat_penalty: float | None = None
+    presence_penalty: float | None = None
+    frequency_penalty: float | None = None
+    mirostat: int | None = None
+    mirostat_tau: float | None = None
+    mirostat_eta: float | None = None
+    penalize_newline: bool | None = None
+    stop: ta.Sequence[str] | None = None
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class BaseRequest(lang.Abstract):
+    model: str
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class BaseStreamableRequest(BaseRequest, lang.Abstract):
+    stream: bool | None = None
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class BaseGenerateRequest(BaseStreamableRequest, lang.Abstract):
+    options: Options | None = None
+    format: ta.Literal['', 'json'] | None = None  # TODO: jsonschema
+    keep_alive: float | str | None = None
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class GenerateRequest(BaseGenerateRequest):
+    prompt: str | None = None
+    suffix: str | None = None
+    system: str | None = None
+    template: str | None = None
+    context: ta.Sequence[int] | None = None
+    raw: bool | None = None
+    images: ta.Sequence[bytes] | None = None
+    think: bool | ta.Literal['low', 'medium', 'high'] | None = None
+
+
+#
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class BaseGenerateResponse(lang.Abstract):
+    model: str | None = None
+    created_at: str | None = None
+    done: bool | None = None
+    done_reason: str | None = None
+    total_duration: int | None = None
+    load_duration: int | None = None
+    prompt_eval_count: int | None = None
+    prompt_eval_duration: int | None = None
+    eval_count: int | None = None
+    eval_duration: int | None = None
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class GenerateResponse(BaseGenerateResponse):
+    response: str
+    thinking: str | None = None
+    context: ta.Sequence[int] | None = None
+
+
+##
+
+
+Role: ta.TypeAlias = ta.Literal[
+    'system',
+    'user',
+    'assistant',
+    'tool',
+]
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class Message:
+    role: Role
+    content: str | None = None
+    thinking: str | None = None
+    images: ta.Sequence[bytes] | None = None
+    tool_name: str | None = None
+
+    @dc.dataclass(frozen=True, kw_only=True)
+    class ToolCall:
+        @dc.dataclass(frozen=True, kw_only=True)
+        class Function:
+            name: str
+            arguments: ta.Mapping[str, ta.Any]
+
+        function: Function
+
+    tool_calls: ta.Sequence[ToolCall] | None = None
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class Tool:
+    type: str | None = 'function'
+
+    @dc.dataclass(frozen=True, kw_only=True)
+    @dc.extra_class_params(default_repr_fn=dc.opt_repr)
+    class Function:
+        name: str | None = None
+        description: str | None = None
+        parameters: ta.Any | None = None
+
+    function: Function | None = None
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class ChatRequest(BaseGenerateRequest):
+    messages: ta.Sequence[Message] | None = None
+    tools: ta.Sequence[Tool] | None = None
+    think: bool | ta.Literal['low', 'medium', 'high'] | None = None
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class ChatResponse(BaseGenerateResponse):
+    message: Message

ommlds/backends/transformers/streamers.py

@@ -0,0 +1,73 @@
+import functools
+import typing as ta
+
+import transformers as tfm
+
+
+T = ta.TypeVar('T')
+P = ta.ParamSpec('P')
+
+
+##
+
+
+class CancellableTextStreamer(tfm.TextStreamer):
+    class Callback(ta.Protocol):
+        def __call__(self, text: str, *, stream_end: bool) -> None: ...
+
+    def __init__(
+            self,
+            tokenizer: tfm.AutoTokenizer,
+            callback: Callback,
+            *,
+            skip_prompt: bool = False,
+            **decode_kwargs: ta.Any,
+    ) -> None:
+        super().__init__(
+            tokenizer,
+            skip_prompt=skip_prompt,
+            **decode_kwargs,
+        )
+
+        self.callback = callback
+
+    _cancelled: bool = False
+
+    #
+
+    @property
+    def cancelled(self) -> bool:
+        return self._cancelled
+
+    def cancel(self) -> None:
+        self._cancelled = True
+
+    class Cancelled(BaseException):  # noqa
+        pass
+
+    @staticmethod
+    def ignoring_cancelled(fn: ta.Callable[P, T]) -> ta.Callable[P, T | None]:
+        @functools.wraps(fn)
+        def inner(*args, **kwargs):
+            try:
+                return fn(*args, **kwargs)
+            except CancellableTextStreamer.Cancelled:
+                pass
+
+        return inner
+
+    def _maybe_raise_cancelled(self) -> None:
+        if self._cancelled:
+            raise CancellableTextStreamer.Cancelled
+
+    #
+
+    def put(self, value: ta.Any) -> None:
+        self._maybe_raise_cancelled()
+        super().put(value)
+        self._maybe_raise_cancelled()
+
+    def on_finalized_text(self, text: str, stream_end: bool = False) -> None:
+        self._maybe_raise_cancelled()
+        self.callback(text, stream_end=stream_end)
+        self._maybe_raise_cancelled()

ommlds/minichain/__init__.py

@@ -558,6 +558,10 @@ with _lang.auto_proxy_init(
     )
 
     from .standard import (  # noqa
+        Device,
+
+        ApiUrl,
+
         ApiKey,
 
         DefaultOptions,

ommlds/minichain/backends/impls/llamacpp/chat.py

@@ -30,6 +30,15 @@ from .format import get_msg_content
 ##
 
 
+# @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
+#     ['ChatChoicesService'],
+#     'llamacpp',
+# )
+
+
+##
+
+
 # @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
 #     name='llamacpp',
 #     type='ChatChoicesService',

ommlds/minichain/backends/impls/llamacpp/stream.py

@@ -29,6 +29,15 @@ from .format import get_msg_content
 ##
 
 
+# @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
+#     ['ChatChoicesStreamService'],
+#     'llamacpp',
+# )
+
+
+##
+
+
 # @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
 #     name='llamacpp',
 #     type='ChatChoicesStreamService',
@@ -76,18 +85,25 @@ class LlamacppChatChoicesStreamService(lang.ExitStacked):
             rs.enter_context(lang.defer(close_output))
 
             async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs] | None:
+                last_role: ta.Any = None
+
                 for chunk in output:
                     check.state(chunk['object'] == 'chat.completion.chunk')
-                    l: list[AiChoiceDeltas] = []
-                    for choice in chunk['choices']:
-                        # FIXME: check role is assistant
-                        # FIXME: stop reason
-                        if not (delta := choice.get('delta', {})):
-                            continue
-                        if not (content := delta.get('content', '')):
-                            continue
-                        l.append(AiChoiceDeltas([ContentAiChoiceDelta(content)]))
-                    await sink.emit(AiChoicesDeltas(l))
+
+                    choice = check.single(chunk['choices'])
+
+                    if not (delta := choice.get('delta', {})):
+                        continue
+
+                    # FIXME: check role is assistant
+                    if (role := delta.get('role')) != last_role:
+                        last_role = role
+
+                    # FIXME: stop reason
+
+                    if (content := delta.get('content', '')):
+                        await sink.emit(AiChoicesDeltas([AiChoiceDeltas([ContentAiChoiceDelta(content)])]))
+
                 return None
 
             return await new_stream_response(rs, inner)

ommlds/minichain/backends/impls/ollama/chat.py

@@ -0,0 +1,196 @@
+import typing as ta
+
+from omlish import check
+from omlish import lang
+from omlish import marshal as msh
+from omlish import typedvalues as tv
+from omlish.formats import json
+from omlish.http import all as http
+from omlish.io.buffers import DelimitingBuffer
+
+from .....backends.ollama import protocol as pt
+from ....chat.choices.services import ChatChoicesOutputs
+from ....chat.choices.services import ChatChoicesRequest
+from ....chat.choices.services import ChatChoicesResponse
+from ....chat.choices.services import static_check_is_chat_choices_service
+from ....chat.choices.types import AiChoice
+from ....chat.messages import AiMessage
+from ....chat.messages import AnyAiMessage
+from ....chat.messages import Message
+from ....chat.messages import SystemMessage
+from ....chat.messages import UserMessage
+from ....chat.stream.services import ChatChoicesStreamRequest
+from ....chat.stream.services import ChatChoicesStreamResponse
+from ....chat.stream.services import static_check_is_chat_choices_stream_service
+from ....chat.stream.types import AiChoiceDeltas
+from ....chat.stream.types import AiChoicesDeltas
+from ....chat.stream.types import ContentAiChoiceDelta
+from ....models.configs import ModelName
+from ....resources import UseResources
+from ....standard import ApiUrl
+from ....stream.services import StreamResponseSink
+from ....stream.services import new_stream_response
+
+
+##
+
+
+# @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
+#     [
+#         'ChatChoicesService',
+#         'ChatChoicesStreamService',
+#     ],
+#     'ollama',
+# )
+
+
+##
+
+
+class BaseOllamaChatChoicesService(lang.Abstract):
+    DEFAULT_API_URL: ta.ClassVar[ApiUrl] = ApiUrl('http://localhost:11434/api')
+    DEFAULT_MODEL_NAME: ta.ClassVar[ModelName] = ModelName('llama3.2')
+
+    def __init__(
+            self,
+            *configs: ApiUrl | ModelName,
+    ) -> None:
+        super().__init__()
+
+        with tv.consume(*configs) as cc:
+            self._api_url = cc.pop(self.DEFAULT_API_URL)
+            self._model_name = cc.pop(self.DEFAULT_MODEL_NAME)
+
+    #
+
+    ROLE_MAP: ta.ClassVar[ta.Mapping[type[Message], pt.Role]] = {
+        SystemMessage: 'system',
+        UserMessage: 'user',
+        AiMessage: 'assistant',
+    }
+
+    @classmethod
+    def _get_message_content(cls, m: Message) -> str | None:
+        if isinstance(m, (AiMessage, UserMessage, SystemMessage)):
+            return check.isinstance(m.c, str)
+        else:
+            raise TypeError(m)
+
+    @classmethod
+    def _build_request_messages(cls, mc_msgs: ta.Iterable[Message]) -> ta.Sequence[pt.Message]:
+        messages: list[pt.Message] = []
+        for m in mc_msgs:
+            messages.append(pt.Message(
+                role=cls.ROLE_MAP[type(m)],
+                content=cls._get_message_content(m),
+            ))
+        return messages
+
+
+##
+
+
+# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
+#     name='ollama',
+#     type='ChatChoicesService',
+# )
+@static_check_is_chat_choices_service
+class OllamaChatChoicesService(BaseOllamaChatChoicesService):
+    async def invoke(
+            self,
+            request: ChatChoicesRequest,
+    ) -> ChatChoicesResponse:
+        messages = self._build_request_messages(request.v)
+
+        a_req = pt.ChatRequest(
+            model=self._model_name.v,
+            messages=messages,
+            # tools=tools or None,
+            stream=False,
+        )
+
+        raw_request = msh.marshal(a_req)
+
+        raw_response = http.request(
+            self._api_url.v.removesuffix('/') + '/chat',
+            data=json.dumps(raw_request).encode('utf-8'),
+        )
+
+        json_response = json.loads(check.not_none(raw_response.data).decode('utf-8'))
+
+        resp = msh.unmarshal(json_response, pt.ChatResponse)
+
+        out: list[AnyAiMessage] = []
+        if resp.message.role == 'assistant':
+            out.append(AiMessage(
+                check.not_none(resp.message.content),
+            ))
+        else:
+            raise TypeError(resp.message.role)
+
+        return ChatChoicesResponse([
+            AiChoice(out),
+        ])
+
+
+##
+
+
+# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
+#     name='ollama',
+#     type='ChatChoicesStreamService',
+# )
+@static_check_is_chat_choices_stream_service
+class OllamaChatChoicesStreamService(BaseOllamaChatChoicesService):
+    READ_CHUNK_SIZE = 64 * 1024
+
+    async def invoke(
+            self,
+            request: ChatChoicesStreamRequest,
+    ) -> ChatChoicesStreamResponse:
+        messages = self._build_request_messages(request.v)
+
+        a_req = pt.ChatRequest(
+            model=self._model_name.v,
+            messages=messages,
+            # tools=tools or None,
+            stream=True,
+        )
+
+        raw_request = msh.marshal(a_req)
+
+        http_request = http.HttpRequest(
+            self._api_url.v.removesuffix('/') + '/chat',
+            data=json.dumps(raw_request).encode('utf-8'),
+        )
+
+        async with UseResources.or_new(request.options) as rs:
+            http_client = rs.enter_context(http.client())
+            http_response = rs.enter_context(http_client.stream_request(http_request))
+
+            async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs] | None:
+                db = DelimitingBuffer([b'\r', b'\n', b'\r\n'])
+                while True:
+                    # FIXME: read1 not on response stream protocol
+                    b = http_response.stream.read1(self.READ_CHUNK_SIZE)  # type: ignore[attr-defined]
+                    for l in db.feed(b):
+                        if isinstance(l, DelimitingBuffer.Incomplete):
+                            # FIXME: handle
+                            return []
+
+                        lj = json.loads(l.decode('utf-8'))
+                        lp: pt.ChatResponse = msh.unmarshal(lj, pt.ChatResponse)
+
+                        check.state(lp.message.role == 'assistant')
+                        check.none(lp.message.tool_name)
+                        check.state(not lp.message.tool_calls)
+
+                        if (c := lp.message.content):
+                            await sink.emit(AiChoicesDeltas([AiChoiceDeltas([ContentAiChoiceDelta(
+                                c,
+                            )])]))
+
+                    if not b:
+                        return []
+
+            return await new_stream_response(rs, inner)

ommlds/minichain/backends/impls/transformers/transformers.py

@@ -16,23 +16,42 @@ from ....chat.choices.services import ChatChoicesRequest
 from ....chat.choices.services import ChatChoicesResponse
 from ....chat.choices.services import static_check_is_chat_choices_service
 from ....chat.choices.types import AiChoice
+from ....chat.choices.types import ChatChoicesOutputs
 from ....chat.messages import AiMessage
 from ....chat.messages import Message
 from ....chat.messages import SystemMessage
 from ....chat.messages import ToolUseMessage
 from ....chat.messages import ToolUseResultMessage
 from ....chat.messages import UserMessage
+from ....chat.stream.services import ChatChoicesStreamRequest
+from ....chat.stream.services import ChatChoicesStreamResponse
+from ....chat.stream.services import static_check_is_chat_choices_stream_service
+from ....chat.stream.types import AiChoiceDeltas  # noqa
+from ....chat.stream.types import AiChoicesDeltas  # noqa
+from ....chat.stream.types import ContentAiChoiceDelta  # noqa
 from ....completion import CompletionRequest
 from ....completion import CompletionResponse
 from ....completion import static_check_is_completion_service
 from ....configs import Config
 from ....models.configs import ModelPath
+from ....resources import UseResources
+from ....stream.services import StreamResponseSink
+from ....stream.services import new_stream_response
 from ...impls.huggingface.configs import HuggingfaceHubToken
 
 
 ##
 
 
+# @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
+#     ['ChatChoicesService', 'ChatChoicesStreamService'],
+#     'transformers',
+# )
+
+
+##
+
+
 class TransformersPipelineKwargs(Config, tv.ScalarTypedValue[ta.Mapping[str, ta.Any]]):
     pass
 
@@ -128,13 +147,10 @@ def build_chat_message(m: Message) -> ta.Mapping[str, ta.Any]:
         raise TypeError(m)
 
 
-# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
-#     name='transformers',
-#     aliases=['tfm'],
-#     type='ChatChoicesService',
-# )
-@static_check_is_chat_choices_service
-class TransformersChatChoicesService(lang.ExitStacked):
+##
+
+
+class BaseTransformersChatChoicesService(lang.ExitStacked):
     DEFAULT_MODEL: ta.ClassVar[str] = (
         'meta-llama/Llama-3.2-1B-Instruct'
     )
@@ -166,16 +182,79 @@ class TransformersChatChoicesService(lang.ExitStacked):
             **pkw,
         )
 
+
+##
+
+
+# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
+#     name='transformers',
+#     aliases=['tfm'],
+#     type='ChatChoicesService',
+# )
+@static_check_is_chat_choices_service
+class TransformersChatChoicesService(BaseTransformersChatChoicesService):
     async def invoke(self, request: ChatChoicesRequest) -> ChatChoicesResponse:
         check.empty(request.options)
 
         pipeline = self._load_pipeline()
 
-        output = pipeline(
-            [
-                build_chat_message(m)
-                for m in request.v
-            ],
-        )
+        inputs = [
+            build_chat_message(m)
+            for m in request.v
+        ]
+
+        outputs = pipeline(inputs)
+
+        gts = check.single(outputs)['generated_text']
+        ugt, agt = gts
+        check.state(ugt['role'] == 'user')
+        check.state(agt['role'] == 'assistant')
+
+        return ChatChoicesResponse([AiChoice([AiMessage(agt['content'])])])
+
+
+##
+
+
+# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
+#     name='transformers',
+#     type='ChatChoicesStreamService',
+# )
+@static_check_is_chat_choices_stream_service
+class TransformersChatChoicesStreamService(BaseTransformersChatChoicesService):
+    async def invoke(self, request: ChatChoicesStreamRequest) -> ChatChoicesStreamResponse:
+        check.empty(request.options)
 
-        return ChatChoicesResponse([AiChoice([output])])
+        pipeline = self._load_pipeline()  # noqa
+
+        inputs = [  # noqa
+            build_chat_message(m)
+            for m in request.v
+        ]
+
+        async with UseResources.or_new(request.options) as rs:
+            async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs] | None:
+                # last_role: ta.Any = None
+                #
+                # for chunk in output:
+                #     check.state(chunk['object'] == 'chat.completion.chunk')
+                #
+                #     choice = check.single(chunk['choices'])
+                #
+                #     if not (delta := choice.get('delta', {})):
+                #         continue
+                #
+                #     # FIXME: check role is assistant
+                #     if (role := delta.get('role')) != last_role:
+                #         last_role = role
+                #
+                #     # FIXME: stop reason
+                #
+                #     if (content := delta.get('content', '')):
+                #         await sink.emit(AiChoicesDeltas([AiChoiceDeltas([ContentAiChoiceDelta(content)])]))
+                #
+                # return None
+
+                raise NotImplementedError
+
+            return await new_stream_response(rs, inner)

ommlds/minichain/standard.py

@@ -25,6 +25,13 @@ class Device(tv.UniqueScalarTypedValue[ta.Any], Config):
 ##
 
 
+class ApiUrl(tv.UniqueScalarTypedValue[str], Config):
+    pass
+
+
+##
+
+
 @dc.dataclass(frozen=True)
 class SecretConfig(Config, lang.Abstract):
     v: sec.SecretRefOrStr = dc.field() | sec.secret_field

{ommlds-0.0.0.dev467.dist-info → ommlds-0.0.0.dev468.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ommlds
-Version: 0.0.0.dev467
+Version: 0.0.0.dev468
 Summary: ommlds
 Author: wrmsr
 License-Expression: BSD-3-Clause
@@ -14,8 +14,8 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev467
-Requires-Dist: omlish==0.0.0.dev467
+Requires-Dist: omdev==0.0.0.dev468
+Requires-Dist: omlish==0.0.0.dev468
 Provides-Extra: all
 Requires-Dist: llama-cpp-python~=0.3; extra == "all"
 Requires-Dist: mlx~=0.29; extra == "all"
@@ -26,8 +26,8 @@ Requires-Dist: tokenizers~=0.22; extra == "all"
 Requires-Dist: torch~=2.9; extra == "all"
 Requires-Dist: transformers~=4.57; extra == "all"
 Requires-Dist: sentence-transformers~=5.1; extra == "all"
-Requires-Dist: huggingface-hub~=0.35; extra == "all"
-Requires-Dist: datasets~=4.2; extra == "all"
+Requires-Dist: huggingface-hub~=0.36; extra == "all"
+Requires-Dist: datasets~=4.3; extra == "all"
 Requires-Dist: numpy>=1.26; extra == "all"
 Requires-Dist: pytesseract~=0.3; extra == "all"
 Requires-Dist: rapidocr-onnxruntime~=1.4; extra == "all"
@@ -47,8 +47,8 @@ Requires-Dist: torch~=2.9; extra == "backends"
 Requires-Dist: transformers~=4.57; extra == "backends"
 Requires-Dist: sentence-transformers~=5.1; extra == "backends"
 Provides-Extra: huggingface
-Requires-Dist: huggingface-hub~=0.35; extra == "huggingface"
-Requires-Dist: datasets~=4.2; extra == "huggingface"
+Requires-Dist: huggingface-hub~=0.36; extra == "huggingface"
+Requires-Dist: datasets~=4.3; extra == "huggingface"
 Provides-Extra: numpy
 Requires-Dist: numpy>=1.26; extra == "numpy"
 Provides-Extra: ocr

{ommlds-0.0.0.dev467.dist-info → ommlds-0.0.0.dev468.dist-info}/RECORD

@@ -1,5 +1,5 @@
-ommlds/.omlish-manifests.json,sha256=MyJQsh5T1CMMXcGdxwqI9abQ8-j-ZlRGRluiikbeKRY,18414
-ommlds/__about__.py,sha256=uAJgr2I_m_oZPlV5P8XLFeYpBlEM-DdzeyF6O5OK_qs,1759
+ommlds/.omlish-manifests.json,sha256=u1WF90X6xpzZW21a4h5zzPyP4a3T30V08RjQz5HGABM,21555
+ommlds/__about__.py,sha256=t2rQF0yXpWFcCb2dvgzGR3I35HKGvGSn-EfhaUWVl5s,1759
 ommlds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ommlds/huggingface.py,sha256=JfEyfKOxU3-SY_ojtXBJFNeD-NIuKjvMe3GL3e93wNA,1175
 ommlds/_hacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -37,6 +37,8 @@ ommlds/backends/mlx/tokenization/detokenization/base.py,sha256=Tezf8Anh-w7BxpNQs
 ommlds/backends/mlx/tokenization/detokenization/bpe.py,sha256=cIw6-r-cyXTfZdyfGRgohrElMIqeLKfMRb8R1H_56nY,3659
 ommlds/backends/mlx/tokenization/detokenization/naive.py,sha256=6L-SvphzP1z16cmVB4QC9VraF7khE8ZcvKqIwwFqN6U,1779
 ommlds/backends/mlx/tokenization/detokenization/spm.py,sha256=IYSnEm-C0z_o5TKLJE_Rj6P0nNd-prT6psVPKsERWAE,1751
+ommlds/backends/ollama/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ommlds/backends/ollama/protocol.py,sha256=1rBZOIb080MsWMfgU4d59wDQhW5EiyBYKgnFbBnLatg,4437
 ommlds/backends/openai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ommlds/backends/openai/protocol/__init__.py,sha256=gYPUQ94GfoIAgU5TGoaC0OVGkWuplrHg-s83ynT9f-4,1750
 ommlds/backends/openai/protocol/_common.py,sha256=r4EXmw1fBFHjU5vbWTDvlM_fsafdIVg3d3PNw4F9m-Q,313
@@ -75,6 +77,8 @@ ommlds/backends/torch/__init__.py,sha256=Id8dKbxMLlp3ux62ohu9JKoXPSrM0ZXUK0eCDTY
 ommlds/backends/torch/backends.py,sha256=Bo-ZdW1n9NswvptT8bL9CssEOKwusDuBMaXVjRS8zrA,3528
 ommlds/backends/torch/devices.py,sha256=KWkeyArPdUwVqckQTJPkN-4GQdv39cpOgCMv_XfkLkQ,776
 ommlds/backends/torch/purge.py,sha256=sp6XUxNLoVCepxIPKw3tevHn-cQqgorILvIQzixauiI,1834
+ommlds/backends/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ommlds/backends/transformers/streamers.py,sha256=Hu_9lp_kUilKjOfs7Ixqr2NoA5FuRn2eRh8JdvaBDYc,1688
 ommlds/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ommlds/cli/__main__.py,sha256=1ffCb0fcUOJMzxROJmJRXQ8PSOVYv7KrcuBtT95cf0c,140
 ommlds/cli/inject.py,sha256=WhTDabJz9b1NRRHVH-UyVN5nj6UncvIeTvgkGrcE9vc,666
@@ -147,7 +151,7 @@ ommlds/cli/state/storage.py,sha256=tRPmgCANRrw7A5Qr700OaH58F6S96O37I8Ivrbo7_gI,3
 ommlds/datasets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ommlds/datasets/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ommlds/datasets/lib/movies.py,sha256=LmdfoXsZU9XMM_r-sxCLv_s06BFzwWO4xUj6sc9XVcI,1961
-ommlds/minichain/__init__.py,sha256=EqNJpuMwqkkdlNmipjaoC30yAqH7c8oziszlkCcXBrQ,10982
+ommlds/minichain/__init__.py,sha256=5S2GfZW4qWF1fin2Ee8YHT4XuD_vEvtCVWTIXHOGrwo,11016
 ommlds/minichain/_marshal.py,sha256=n9PGWrHhvAmGIc7KDOYt3IF9Z6G0ncXskyICTp3Ji6k,1923
 ommlds/minichain/_typedvalues.py,sha256=Vl1Edt5khC0e5RPFBPmPCxn0IzrfVd0NHzAjAN2E6Kc,2183
 ommlds/minichain/completion.py,sha256=lQ0LfCIYZsvDqteHhhDIv16D2_gn_xMfEL0ouywE5Yo,1033
@@ -157,7 +161,7 @@ ommlds/minichain/json.py,sha256=0_5rV5Zi2qPOvXi2CLAc5DF7FN3jK3ABbjoKdjtTuVo,360
 ommlds/minichain/metadata.py,sha256=2jik8gEm_VMnknPuPwqRssTg0MClRFUrXz_IsyEgUt4,878
 ommlds/minichain/resources.py,sha256=HfcydnyFmXVRspYw-32-lvM_OfrZQdPEebAt3ivLev0,4436
 ommlds/minichain/search.py,sha256=azRzWcYhcm9IgSHquqLwtbwowtYCRAtPLSm7Gvt9iNo,1262
-ommlds/minichain/standard.py,sha256=uKXvdUNLxdUu7suCBsVOjJtnYVC2hjD_tmz3Ra7H6Jg,2510
+ommlds/minichain/standard.py,sha256=cGXaGtC5iM9Q2lCcbhLtvEcPGKhcJUIh3UWyNgOssRM,2580
 ommlds/minichain/types.py,sha256=K6RRjpUi17UEG0cqPrrvbVANU0iRVh3WLiH-y6oEWFI,414
 ommlds/minichain/utils.py,sha256=NTsBu_pSZnLdZc1R1Se70rb_9J-IoB6VRwjhwzh3PwY,490
 ommlds/minichain/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -185,12 +189,14 @@ ommlds/minichain/backends/impls/huggingface/__init__.py,sha256=47DEQpj8HBSa-_TIm
 ommlds/minichain/backends/impls/huggingface/configs.py,sha256=6jsBtPNXOP57PcpxNTVLGWLc-18Iwn_lDbGouwCJTIQ,258
 ommlds/minichain/backends/impls/huggingface/repos.py,sha256=8BDxJmra9elSQL2vzp2nr2p4Hpq56A3zTk7hTTnfJU4,861
 ommlds/minichain/backends/impls/llamacpp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ommlds/minichain/backends/impls/llamacpp/chat.py,sha256=YeBzlA_3gcuF6KF0HIE7abUp28_o1Kil-SujyQNAHyE,5508
+ommlds/minichain/backends/impls/llamacpp/chat.py,sha256=J6Jslx9atAtWvLdrVtvRboQUBzRX7Z5aHlo0dK5X78A,5649
 ommlds/minichain/backends/impls/llamacpp/completion.py,sha256=oJ2I6wUoIPXYLm9Vc7dwOPgqbevatTjNBZ-jXeM24tQ,2372
 ommlds/minichain/backends/impls/llamacpp/format.py,sha256=fcLMwk7r7FbNrYCH39G3fDRInKvlPIqcoxyLj95CooA,778
-ommlds/minichain/backends/impls/llamacpp/stream.py,sha256=uGog3xPNqCjGgyZjXEjhlxKbIbakWbapjANAEsmW-U4,3378
+ommlds/minichain/backends/impls/llamacpp/stream.py,sha256=uzrXr2HhshgFe3Z0g8KTPc6Dr2kPsyxZabIy2d6IOBg,3547
 ommlds/minichain/backends/impls/mlx/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ommlds/minichain/backends/impls/mlx/chat.py,sha256=sMlhgiFZrxAC-kKkLSJ6c-2uJn0IHZXH4EiPET_-CKI,7458
+ommlds/minichain/backends/impls/ollama/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ommlds/minichain/backends/impls/ollama/chat.py,sha256=UK19riOph-ptIz9zW7PucGWvVEtWHOHvwp7hoKurDNw,6393
 ommlds/minichain/backends/impls/openai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ommlds/minichain/backends/impls/openai/chat.py,sha256=eMRjxPNrzrRjaw83LJuYzP9DGvwGyY2ObJSZub4Z9bY,2658
 ommlds/minichain/backends/impls/openai/completion.py,sha256=0XTC08mZzbW23Y2DNW2xfRR0eDX4nTyejF8CR1BdHZs,1756
@@ -207,7 +213,7 @@ ommlds/minichain/backends/impls/tokenizers/tokens.py,sha256=_8Q49k5YroG5wQI0cuK6
 ommlds/minichain/backends/impls/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ommlds/minichain/backends/impls/transformers/sentence.py,sha256=1bFJ-ND3MOkj7mNsPuISrQCpqTs7npmmNmYcc2go-Fk,1393
 ommlds/minichain/backends/impls/transformers/tokens.py,sha256=uS3-IWOJRUMBfPDVRrp3SCaXdE1yzEdKHQcyv0JZQIw,2089
-ommlds/minichain/backends/impls/transformers/transformers.py,sha256=Bb1RnvDlo8bzu24ByhDacDC0sN7R7KYZnPZ9hjbViBg,5287
+ommlds/minichain/backends/impls/transformers/transformers.py,sha256=U4O-MiVH3dRXf-UNSoKZueZVM8XvAm2mMr30qQUHhFY,8000
 ommlds/minichain/backends/strings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ommlds/minichain/backends/strings/manifests.py,sha256=kmlanVUAZqIh0P95Mm8H20e8ib3gEgYHHUlkCXDQGFk,413
 ommlds/minichain/backends/strings/parsing.py,sha256=2wChk9Z8fhqJTk8_91f8QFjKcSZygOQM_rVk-P4NnKw,1772
@@ -367,9 +373,9 @@ ommlds/wiki/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 ommlds/wiki/utils/io.py,sha256=UKgDJGtmpnWvIqVd2mJc2QNPOqlToEY1GEveNp6_pMo,7088
 ommlds/wiki/utils/progress.py,sha256=EhvKcMFYtsarCQhIahlO6f0SboyAKP3UwUyrnVnP-Vk,3222
 ommlds/wiki/utils/xml.py,sha256=vVV8Ctn13aaRM9eYfs9Wd6rHn5WOCEUzQ44fIhOvJdg,3754
-ommlds-0.0.0.dev467.dist-info/licenses/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-ommlds-0.0.0.dev467.dist-info/METADATA,sha256=NvYqf0PtfEdrj2en5RFFpTHB_cGjm41groVHgy54WZQ,3224
-ommlds-0.0.0.dev467.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ommlds-0.0.0.dev467.dist-info/entry_points.txt,sha256=Z5YWtX7ClfiCKdW-dd_CSVvM0h4yQpJPi-2G3q6gNFo,35
-ommlds-0.0.0.dev467.dist-info/top_level.txt,sha256=Rbnk5d5wi58vnAXx13WFZqdQ4VX8hBCS2hEL3WeXOhY,7
-ommlds-0.0.0.dev467.dist-info/RECORD,,
+ommlds-0.0.0.dev468.dist-info/licenses/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+ommlds-0.0.0.dev468.dist-info/METADATA,sha256=k1H1yGwCqmZETx1eTEKBsNZ_whn2QWxWap9pdErGIkw,3224
+ommlds-0.0.0.dev468.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ommlds-0.0.0.dev468.dist-info/entry_points.txt,sha256=Z5YWtX7ClfiCKdW-dd_CSVvM0h4yQpJPi-2G3q6gNFo,35
+ommlds-0.0.0.dev468.dist-info/top_level.txt,sha256=Rbnk5d5wi58vnAXx13WFZqdQ4VX8hBCS2hEL3WeXOhY,7
+ommlds-0.0.0.dev468.dist-info/RECORD,,