ommlds 0.0.0.dev466__py3-none-any.whl → 0.0.0.dev468__py3-none-any.whl

ommlds/.omlish-manifests.json

@@ -170,6 +170,21 @@
     "attr": null,
     "file": "ommlds/minichain/backends/impls/llamacpp/chat.py",
     "line": 33,
+    "value": {
+      "!.minichain.backends.strings.manifests.BackendStringsManifest": {
+        "service_cls_names": [
+          "ChatChoicesService"
+        ],
+        "backend_name": "llamacpp",
+        "model_names": null
+      }
+    }
+  },
+  {
+    "module": ".minichain.backends.impls.llamacpp.chat",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/llamacpp/chat.py",
+    "line": 42,
     "value": {
       "!.minichain.registries.manifests.RegistryManifest": {
         "module": "ommlds.minichain.backends.impls.llamacpp.chat",
@@ -200,6 +215,21 @@
     "attr": null,
     "file": "ommlds/minichain/backends/impls/llamacpp/stream.py",
     "line": 32,
+    "value": {
+      "!.minichain.backends.strings.manifests.BackendStringsManifest": {
+        "service_cls_names": [
+          "ChatChoicesStreamService"
+        ],
+        "backend_name": "llamacpp",
+        "model_names": null
+      }
+    }
+  },
+  {
+    "module": ".minichain.backends.impls.llamacpp.stream",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/llamacpp/stream.py",
+    "line": 41,
     "value": {
       "!.minichain.registries.manifests.RegistryManifest": {
         "module": "ommlds.minichain.backends.impls.llamacpp.stream",
@@ -229,11 +259,12 @@
     "module": ".minichain.backends.impls.mlx.chat",
     "attr": null,
     "file": "ommlds/minichain/backends/impls/mlx/chat.py",
-    "line": 28,
+    "line": 39,
     "value": {
       "!.minichain.backends.strings.manifests.BackendStringsManifest": {
         "service_cls_names": [
-          "ChatChoicesService"
+          "ChatChoicesService",
+          "ChatChoicesStreamService"
         ],
         "backend_name": "mlx",
         "model_names": null
@@ -244,7 +275,7 @@
     "module": ".minichain.backends.impls.mlx.chat",
     "attr": null,
     "file": "ommlds/minichain/backends/impls/mlx/chat.py",
-    "line": 37,
+    "line": 133,
     "value": {
       "!.minichain.registries.manifests.RegistryManifest": {
         "module": "ommlds.minichain.backends.impls.mlx.chat",
@@ -255,6 +286,67 @@
       }
     }
   },
+  {
+    "module": ".minichain.backends.impls.mlx.chat",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/mlx/chat.py",
+    "line": 164,
+    "value": {
+      "!.minichain.registries.manifests.RegistryManifest": {
+        "module": "ommlds.minichain.backends.impls.mlx.chat",
+        "attr": "MlxChatChoicesStreamService",
+        "name": "mlx",
+        "aliases": null,
+        "type": "ChatChoicesStreamService"
+      }
+    }
+  },
+  {
+    "module": ".minichain.backends.impls.ollama.chat",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/ollama/chat.py",
+    "line": 38,
+    "value": {
+      "!.minichain.backends.strings.manifests.BackendStringsManifest": {
+        "service_cls_names": [
+          "ChatChoicesService",
+          "ChatChoicesStreamService"
+        ],
+        "backend_name": "ollama",
+        "model_names": null
+      }
+    }
+  },
+  {
+    "module": ".minichain.backends.impls.ollama.chat",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/ollama/chat.py",
+    "line": 93,
+    "value": {
+      "!.minichain.registries.manifests.RegistryManifest": {
+        "module": "ommlds.minichain.backends.impls.ollama.chat",
+        "attr": "OllamaChatChoicesService",
+        "name": "ollama",
+        "aliases": null,
+        "type": "ChatChoicesService"
+      }
+    }
+  },
+  {
+    "module": ".minichain.backends.impls.ollama.chat",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/ollama/chat.py",
+    "line": 139,
+    "value": {
+      "!.minichain.registries.manifests.RegistryManifest": {
+        "module": "ommlds.minichain.backends.impls.ollama.chat",
+        "attr": "OllamaChatChoicesStreamService",
+        "name": "ollama",
+        "aliases": null,
+        "type": "ChatChoicesStreamService"
+      }
+    }
+  },
   {
     "module": ".minichain.backends.impls.openai.chat",
     "attr": null,
@@ -356,7 +448,7 @@
     "module": ".minichain.backends.impls.openai.stream",
     "attr": null,
     "file": "ommlds/minichain/backends/impls/openai/stream.py",
-    "line": 37,
+    "line": 38,
     "value": {
       "!.minichain.registries.manifests.RegistryManifest": {
         "module": "ommlds.minichain.backends.impls.openai.stream",
@@ -434,7 +526,23 @@
     "module": ".minichain.backends.impls.transformers.transformers",
     "attr": null,
     "file": "ommlds/minichain/backends/impls/transformers/transformers.py",
-    "line": 43,
+    "line": 46,
+    "value": {
+      "!.minichain.backends.strings.manifests.BackendStringsManifest": {
+        "service_cls_names": [
+          "ChatChoicesService",
+          "ChatChoicesStreamService"
+        ],
+        "backend_name": "transformers",
+        "model_names": null
+      }
+    }
+  },
+  {
+    "module": ".minichain.backends.impls.transformers.transformers",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/transformers/transformers.py",
+    "line": 62,
     "value": {
       "!.minichain.registries.manifests.RegistryManifest": {
         "module": "ommlds.minichain.backends.impls.transformers.transformers",
@@ -451,7 +559,7 @@
     "module": ".minichain.backends.impls.transformers.transformers",
     "attr": null,
     "file": "ommlds/minichain/backends/impls/transformers/transformers.py",
-    "line": 131,
+    "line": 189,
     "value": {
       "!.minichain.registries.manifests.RegistryManifest": {
         "module": "ommlds.minichain.backends.impls.transformers.transformers",
@@ -464,6 +572,21 @@
       }
     }
   },
+  {
+    "module": ".minichain.backends.impls.transformers.transformers",
+    "attr": null,
+    "file": "ommlds/minichain/backends/impls/transformers/transformers.py",
+    "line": 219,
+    "value": {
+      "!.minichain.registries.manifests.RegistryManifest": {
+        "module": "ommlds.minichain.backends.impls.transformers.transformers",
+        "attr": "TransformersChatChoicesStreamService",
+        "name": "transformers",
+        "aliases": null,
+        "type": "ChatChoicesStreamService"
+      }
+    }
+  },
   {
     "module": ".minichain.chat.choices.services",
     "attr": null,

ommlds/__about__.py

@@ -37,8 +37,8 @@ class Project(ProjectBase):
         ],
 
         'huggingface': [
-            'huggingface-hub ~= 0.35',
-            'datasets ~= 4.2',
+            'huggingface-hub ~= 0.36',
+            'datasets ~= 4.3',
         ],
 
         'numpy': [

ommlds/backends/ollama/protocol.py

@@ -0,0 +1,170 @@
+"""
+https://docs.ollama.com/api
+"""
+import typing as ta
+
+from omlish import dataclasses as dc
+from omlish import lang
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class Options:
+    # loading
+    numa: bool | None = None
+    num_ctx: int | None = None
+    num_batch: int | None = None
+    num_gpu: int | None = None
+    main_gpu: int | None = None
+    low_vram: bool | None = None
+    f16_kv: bool | None = None
+    logits_all: bool | None = None
+    vocab_only: bool | None = None
+    use_mmap: bool | None = None
+    use_mlock: bool | None = None
+    embedding_only: bool | None = None
+    num_thread: int | None = None
+
+    # querying
+    num_keep: int | None = None
+    seed: int | None = None
+    num_predict: int | None = None
+    top_k: int | None = None
+    top_p: float | None = None
+    tfs_z: float | None = None
+    typical_p: float | None = None
+    repeat_last_n: int | None = None
+    temperature: float | None = None
+    repeat_penalty: float | None = None
+    presence_penalty: float | None = None
+    frequency_penalty: float | None = None
+    mirostat: int | None = None
+    mirostat_tau: float | None = None
+    mirostat_eta: float | None = None
+    penalize_newline: bool | None = None
+    stop: ta.Sequence[str] | None = None
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class BaseRequest(lang.Abstract):
+    model: str
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class BaseStreamableRequest(BaseRequest, lang.Abstract):
+    stream: bool | None = None
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class BaseGenerateRequest(BaseStreamableRequest, lang.Abstract):
+    options: Options | None = None
+    format: ta.Literal['', 'json'] | None = None  # TODO: jsonschema
+    keep_alive: float | str | None = None
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class GenerateRequest(BaseGenerateRequest):
+    prompt: str | None = None
+    suffix: str | None = None
+    system: str | None = None
+    template: str | None = None
+    context: ta.Sequence[int] | None = None
+    raw: bool | None = None
+    images: ta.Sequence[bytes] | None = None
+    think: bool | ta.Literal['low', 'medium', 'high'] | None = None
+
+
+#
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class BaseGenerateResponse(lang.Abstract):
+    model: str | None = None
+    created_at: str | None = None
+    done: bool | None = None
+    done_reason: str | None = None
+    total_duration: int | None = None
+    load_duration: int | None = None
+    prompt_eval_count: int | None = None
+    prompt_eval_duration: int | None = None
+    eval_count: int | None = None
+    eval_duration: int | None = None
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class GenerateResponse(BaseGenerateResponse):
+    response: str
+    thinking: str | None = None
+    context: ta.Sequence[int] | None = None
+
+
+##
+
+
+Role: ta.TypeAlias = ta.Literal[
+    'system',
+    'user',
+    'assistant',
+    'tool',
+]
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class Message:
+    role: Role
+    content: str | None = None
+    thinking: str | None = None
+    images: ta.Sequence[bytes] | None = None
+    tool_name: str | None = None
+
+    @dc.dataclass(frozen=True, kw_only=True)
+    class ToolCall:
+        @dc.dataclass(frozen=True, kw_only=True)
+        class Function:
+            name: str
+            arguments: ta.Mapping[str, ta.Any]
+
+        function: Function
+
+    tool_calls: ta.Sequence[ToolCall] | None = None
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class Tool:
+    type: str | None = 'function'
+
+    @dc.dataclass(frozen=True, kw_only=True)
+    @dc.extra_class_params(default_repr_fn=dc.opt_repr)
+    class Function:
+        name: str | None = None
+        description: str | None = None
+        parameters: ta.Any | None = None
+
+    function: Function | None = None
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class ChatRequest(BaseGenerateRequest):
+    messages: ta.Sequence[Message] | None = None
+    tools: ta.Sequence[Tool] | None = None
+    think: bool | ta.Literal['low', 'medium', 'high'] | None = None
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@dc.extra_class_params(default_repr_fn=dc.opt_repr)
+class ChatResponse(BaseGenerateResponse):
+    message: Message

ommlds/backends/transformers/streamers.py

@@ -0,0 +1,73 @@
+import functools
+import typing as ta
+
+import transformers as tfm
+
+
+T = ta.TypeVar('T')
+P = ta.ParamSpec('P')
+
+
+##
+
+
+class CancellableTextStreamer(tfm.TextStreamer):
+    class Callback(ta.Protocol):
+        def __call__(self, text: str, *, stream_end: bool) -> None: ...
+
+    def __init__(
+            self,
+            tokenizer: tfm.AutoTokenizer,
+            callback: Callback,
+            *,
+            skip_prompt: bool = False,
+            **decode_kwargs: ta.Any,
+    ) -> None:
+        super().__init__(
+            tokenizer,
+            skip_prompt=skip_prompt,
+            **decode_kwargs,
+        )
+
+        self.callback = callback
+
+    _cancelled: bool = False
+
+    #
+
+    @property
+    def cancelled(self) -> bool:
+        return self._cancelled
+
+    def cancel(self) -> None:
+        self._cancelled = True
+
+    class Cancelled(BaseException):  # noqa
+        pass
+
+    @staticmethod
+    def ignoring_cancelled(fn: ta.Callable[P, T]) -> ta.Callable[P, T | None]:
+        @functools.wraps(fn)
+        def inner(*args, **kwargs):
+            try:
+                return fn(*args, **kwargs)
+            except CancellableTextStreamer.Cancelled:
+                pass
+
+        return inner
+
+    def _maybe_raise_cancelled(self) -> None:
+        if self._cancelled:
+            raise CancellableTextStreamer.Cancelled
+
+    #
+
+    def put(self, value: ta.Any) -> None:
+        self._maybe_raise_cancelled()
+        super().put(value)
+        self._maybe_raise_cancelled()
+
+    def on_finalized_text(self, text: str, stream_end: bool = False) -> None:
+        self._maybe_raise_cancelled()
+        self.callback(text, stream_end=stream_end)
+        self._maybe_raise_cancelled()

ommlds/cli/sessions/chat/backends/catalog.py

@@ -32,7 +32,7 @@ class _CatalogBackendProvider(BackendProvider[ServiceT], lang.Abstract):
     @contextlib.asynccontextmanager
     async def _provide_backend(self, cls: type[ServiceT]) -> ta.AsyncIterator[ServiceT]:
         service: ServiceT
-        async with lang.async_maybe_managing(self._catalog.get_backend(
+        async with lang.async_or_sync_maybe_managing(self._catalog.get_backend(
                 cls,
                 self._name,
                 *(self._configs or []),

ommlds/minichain/__init__.py

@@ -558,6 +558,10 @@ with _lang.auto_proxy_init(
     )
 
     from .standard import (  # noqa
+        Device,
+
+        ApiUrl,
+
         ApiKey,
 
         DefaultOptions,

ommlds/minichain/backends/impls/llamacpp/chat.py

@@ -30,6 +30,15 @@ from .format import get_msg_content
 ##
 
 
+# @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
+#     ['ChatChoicesService'],
+#     'llamacpp',
+# )
+
+
+##
+
+
 # @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
 #     name='llamacpp',
 #     type='ChatChoicesService',

ommlds/minichain/backends/impls/llamacpp/stream.py

@@ -29,6 +29,15 @@ from .format import get_msg_content
 ##
 
 
+# @omlish-manifest $.minichain.backends.strings.manifests.BackendStringsManifest(
+#     ['ChatChoicesStreamService'],
+#     'llamacpp',
+# )
+
+
+##
+
+
 # @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
 #     name='llamacpp',
 #     type='ChatChoicesStreamService',
@@ -76,18 +85,25 @@ class LlamacppChatChoicesStreamService(lang.ExitStacked):
             rs.enter_context(lang.defer(close_output))
 
             async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs] | None:
+                last_role: ta.Any = None
+
                 for chunk in output:
                     check.state(chunk['object'] == 'chat.completion.chunk')
-                    l: list[AiChoiceDeltas] = []
-                    for choice in chunk['choices']:
-                        # FIXME: check role is assistant
-                        # FIXME: stop reason
-                        if not (delta := choice.get('delta', {})):
-                            continue
-                        if not (content := delta.get('content', '')):
-                            continue
-                        l.append(AiChoiceDeltas([ContentAiChoiceDelta(content)]))
-                    await sink.emit(AiChoicesDeltas(l))
+
+                    choice = check.single(chunk['choices'])
+
+                    if not (delta := choice.get('delta', {})):
+                        continue
+
+                    # FIXME: check role is assistant
+                    if (role := delta.get('role')) != last_role:
+                        last_role = role
+
+                    # FIXME: stop reason
+
+                    if (content := delta.get('content', '')):
+                        await sink.emit(AiChoicesDeltas([AiChoiceDeltas([ContentAiChoiceDelta(content)])]))
+
                 return None
 
             return await new_stream_response(rs, inner)