ommlds 0.0.0.dev466__py3-none-any.whl → 0.0.0.dev512__py3-none-any.whl

ommlds/minichain/backends/impls/openai/stream.py

@@ -9,26 +9,24 @@ from omlish import typedvalues as tv
 from omlish.formats import json
 from omlish.http import all as http
 from omlish.http import sse
-from omlish.io.buffers import DelimitingBuffer
-
-from .....backends.openai.protocol.chatcompletion.chunk import ChatCompletionChunk
-from ....chat.choices.services import ChatChoicesOutputs
-from ....chat.stream.services import ChatChoicesStreamRequest
-from ....chat.stream.services import ChatChoicesStreamResponse
-from ....chat.stream.services import static_check_is_chat_choices_stream_service
-from ....chat.stream.types import AiChoiceDeltas
-from ....chat.stream.types import AiChoicesDeltas
-from ....chat.stream.types import ChatChoicesStreamOption
+
+from .....backends.openai import protocol as pt
+from ....chat.choices.stream.services import ChatChoicesStreamRequest
+from ....chat.choices.stream.services import ChatChoicesStreamResponse
+from ....chat.choices.stream.services import static_check_is_chat_choices_stream_service
+from ....chat.choices.stream.types import AiChoiceDeltas
+from ....chat.choices.stream.types import AiChoicesDeltas
+from ....chat.choices.stream.types import ChatChoicesStreamOption
 from ....configs import Config
+from ....http.stream import BytesHttpStreamResponseBuilder
+from ....http.stream import SimpleSseLinesHttpStreamResponseHandler
 from ....resources import ResourcesOption
-from ....resources import UseResources
 from ....standard import ApiKey
 from ....stream.services import StreamOption
-from ....stream.services import StreamResponseSink
-from ....stream.services import new_stream_response
 from .chat import OpenaiChatChoicesService
 from .format import OpenaiChatRequestHandler
-from .names import MODEL_NAMES
+from .format import build_mc_ai_delta
+from .names import CHAT_MODEL_NAMES
 
 
 ##
@@ -40,18 +38,51 @@ from .names import MODEL_NAMES
 # )
 @static_check_is_chat_choices_stream_service
 class OpenaiChatChoicesStreamService:
-    def __init__(self, *configs: Config) -> None:
+    def __init__(
+            self,
+            *configs: Config,
+            http_client: http.AsyncHttpClient | None = None,
+    ) -> None:
         super().__init__()
 
+        self._http_client = http_client
+
         with tv.consume(*configs) as cc:
             self._model_name = cc.pop(OpenaiChatChoicesService.DEFAULT_MODEL_NAME)
             self._api_key = ApiKey.pop_secret(cc, env='OPENAI_API_KEY')
 
-    READ_CHUNK_SIZE = 64 * 1024
+    URL: ta.ClassVar[str] = 'https://api.openai.com/v1/chat/completions'
 
-    async def invoke(self, request: ChatChoicesStreamRequest) -> ChatChoicesStreamResponse:
-        # check.isinstance(request, ChatRequest)
+    def _process_sse(self, so: sse.SseDecoderOutput) -> ta.Sequence[AiChoicesDeltas | None]:
+        if not (isinstance(so, sse.SseEvent) and so.type == b'message'):
+            return []
+
+        ss = so.data.decode('utf-8')
+        if ss == '[DONE]':
+            return [None]
+
+        sj = json.loads(ss)  # ChatCompletionChunk
+
+        check.state(sj['object'] == 'chat.completion.chunk')
+
+        ccc = msh.unmarshal(sj, pt.ChatCompletionChunk)
 
+        # FIXME: stop reason
+        if not ccc.choices:
+            return []
+
+        if any(choice.finish_reason for choice in ccc.choices):
+            check.state(all(choice.finish_reason for choice in ccc.choices))
+            return [None]
+
+        return [AiChoicesDeltas([
+            AiChoiceDeltas([build_mc_ai_delta(choice.delta)])
+            for choice in ccc.choices
+        ])]
+
+    READ_CHUNK_SIZE: ta.ClassVar[int] = -1
+
+    async def invoke(self, request: ChatChoicesStreamRequest) -> ChatChoicesStreamResponse:
         rh = OpenaiChatRequestHandler(
             request.v,
             *[
@@ -59,19 +90,19 @@ class OpenaiChatChoicesStreamService:
                 for o in request.options
                 if not isinstance(o, (ChatChoicesStreamOption, StreamOption, ResourcesOption))
             ],
-            model=MODEL_NAMES.resolve(self._model_name.v),
+            model=CHAT_MODEL_NAMES.resolve(self._model_name.v),
             mandatory_kwargs=dict(
                 stream=True,
-                stream_options=dict(
+                stream_options=pt.ChatCompletionRequest.StreamOptions(
                     include_usage=True,
                 ),
             ),
         )
 
-        raw_request = rh.raw_request()
+        raw_request = msh.marshal(rh.oai_request())
 
         http_request = http.HttpRequest(
-            'https://api.openai.com/v1/chat/completions',
+            self.URL,
             headers={
                 http.consts.HEADER_CONTENT_TYPE: http.consts.CONTENT_TYPE_JSON,
                 http.consts.HEADER_AUTH: http.consts.format_bearer_auth_header(check.not_none(self._api_key).reveal()),
@@ -79,51 +110,11 @@ class OpenaiChatChoicesStreamService:
             data=json.dumps(raw_request).encode('utf-8'),
         )
 
-        async with UseResources.or_new(request.options) as rs:
-            http_client = rs.enter_context(http.client())
-            http_response = rs.enter_context(http_client.stream_request(http_request))
-
-            async def inner(sink: StreamResponseSink[AiChoicesDeltas]) -> ta.Sequence[ChatChoicesOutputs]:
-                db = DelimitingBuffer([b'\r', b'\n', b'\r\n'])
-                sd = sse.SseDecoder()
-                while True:
-                    # FIXME: read1 not on response stream protocol
-                    b = http_response.stream.read1(self.READ_CHUNK_SIZE)  # type: ignore[attr-defined]
-                    for l in db.feed(b):
-                        if isinstance(l, DelimitingBuffer.Incomplete):
-                            # FIXME: handle
-                            return []
-
-                        # FIXME: https://platform.openai.com/docs/guides/function-calling?api-mode=responses#streaming
-                        for so in sd.process_line(l):
-                            if isinstance(so, sse.SseEvent) and so.type == b'message':
-                                ss = so.data.decode('utf-8')
-                                if ss == '[DONE]':
-                                    return []
-
-                                sj = json.loads(ss)  # ChatCompletionChunk
-
-                                check.state(sj['object'] == 'chat.completion.chunk')
-
-                                ccc = msh.unmarshal(sj, ChatCompletionChunk)  # noqa
-                                # print(ccc)
-
-                                # FIXME: stop reason
-                                if not sj['choices']:
-                                    continue
-
-                                if any(choice['delta'] for choice in sj['choices']):
-                                    await sink.emit(AiChoicesDeltas([
-                                        AiChoiceDeltas(
-                                            [rh.build_ai_choice_delta(choice['delta'])] if choice['delta'] else [],
-                                        )
-                                        for choice in sj['choices']
-                                    ]))
-
-                    if not b:
-                        return []
-
-            # raw_response = json.loads(check.not_none(http_response.data).decode('utf-8'))
-            # return rh.build_response(raw_response)
-
-            return await new_stream_response(rs, inner)
+        return await BytesHttpStreamResponseBuilder(
+            self._http_client,
+            lambda http_response: SimpleSseLinesHttpStreamResponseHandler(self._process_sse).as_lines().as_bytes(),
+            read_chunk_size=self.READ_CHUNK_SIZE,
+        ).new_stream_response(
+            http_request,
+            request.options,
+        )

ommlds/minichain/backends/impls/sentencepiece/tokens.py

@@ -1,23 +1,26 @@
 import typing as ta
 
-import sentencepiece as spm
-
 from omlish import check
+from omlish import lang
 
 from .... import tokens as tks
 
 
+with lang.auto_proxy_import(globals()):
+    import sentencepiece as spm
+
+
 ##
 
 
-def build_vocab(spm_tokenizer: spm.SentencePieceProcessor) -> tks.Vocab:
+def build_vocab(spm_tokenizer: 'spm.SentencePieceProcessor') -> tks.Vocab:
     return tks.Vocab([
         (ta.cast(tks.Token, i), tks.TokenStr(spm_tokenizer.id_to_piece(i)))  # noqa
         for i in range(spm_tokenizer.get_piece_size())  # noqa
     ])
 
 
-def build_specials(spm_tokenizer: spm.SentencePieceProcessor) -> tks.SpecialTokens:
+def build_specials(spm_tokenizer: 'spm.SentencePieceProcessor') -> tks.SpecialTokens:
     # FIXME
     return tks.SpecialTokens([])
 
@@ -28,7 +31,7 @@ def build_specials(spm_tokenizer: spm.SentencePieceProcessor) -> tks.SpecialToke
 class SentencepieceTokenizer(tks.BaseTokenizer):
     def __init__(
             self,
-            spm_tokenizer: spm.SentencePieceProcessor,
+            spm_tokenizer: 'spm.SentencePieceProcessor',
     ) -> None:
         self._spm_tokenizer = check.isinstance(spm_tokenizer, spm.SentencePieceProcessor)
 
@@ -38,7 +41,7 @@ class SentencepieceTokenizer(tks.BaseTokenizer):
         )
 
     @property
-    def spm_tokenizer(self) -> spm.SentencePieceProcessor:
+    def spm_tokenizer(self) -> 'spm.SentencePieceProcessor':
         return self._spm_tokenizer
 
     #

ommlds/minichain/backends/impls/tavily.py

@@ -0,0 +1,66 @@
+from omlish import check
+from omlish import marshal as msh
+from omlish import typedvalues as tv
+from omlish.formats import json
+from omlish.http import all as http
+
+from ....backends.tavily import protocol as pt
+from ...search import SearchHit
+from ...search import SearchHits
+from ...search import SearchRequest
+from ...search import SearchResponse
+from ...search import static_check_is_search_service
+from ...standard import ApiKey
+
+
+##
+
+
+# @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
+#     name='tavily',
+#     type='SearchService',
+# )
+@static_check_is_search_service
+class TavilySearchService:
+    def __init__(
+            self,
+            *configs: ApiKey,
+            http_client: http.AsyncHttpClient | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._http_client = http_client
+
+        with tv.consume(*configs) as cc:
+            self._api_key = ApiKey.pop_secret(cc, env='TAVILY_API_KEY')
+
+    async def invoke(self, request: SearchRequest) -> SearchResponse:
+        pt_request = pt.SearchRequest(
+            query=request.v,
+        )
+
+        raw_request = msh.marshal(pt_request)
+
+        http_response = await http.async_request(
+            'https://api.tavily.com/search',
+            headers={
+                http.consts.HEADER_CONTENT_TYPE: http.consts.CONTENT_TYPE_JSON,
+                http.consts.HEADER_AUTH: http.consts.format_bearer_auth_header(check.not_none(self._api_key).reveal()),
+            },
+            data=json.dumps(raw_request).encode('utf-8'),
+            client=self._http_client,
+        )
+
+        raw_response = json.loads(check.not_none(http_response.data).decode('utf-8'))
+
+        pt_response = msh.unmarshal(raw_response, pt.SearchResponse)
+
+        return SearchResponse(SearchHits(
+            l=[
+                SearchHit(
+                    title=r.title,
+                    url=r.url,
+                )
+                for r in pt_response.results or []
+            ],
+        ))

ommlds/minichain/backends/impls/tinygrad/chat.py

@@ -4,23 +4,22 @@ import typing as ta
 from omlish import check
 from omlish import lang
 
-from .....backends.tinygrad.models import llama3 as tgl3
 from ....chat.choices.services import ChatChoicesOptions
 from ....chat.choices.services import ChatChoicesRequest
 from ....chat.choices.services import ChatChoicesResponse
 from ....chat.choices.services import static_check_is_chat_choices_service
+from ....chat.choices.stream.services import ChatChoicesStreamRequest
+from ....chat.choices.stream.services import ChatChoicesStreamResponse
+from ....chat.choices.stream.services import static_check_is_chat_choices_stream_service
+from ....chat.choices.stream.types import AiChoiceDeltas
+from ....chat.choices.stream.types import AiChoicesDeltas
 from ....chat.choices.types import AiChoice
 from ....chat.choices.types import ChatChoicesOutputs
 from ....chat.messages import AiMessage
 from ....chat.messages import Chat
 from ....chat.messages import SystemMessage
 from ....chat.messages import UserMessage
-from ....chat.stream.services import ChatChoicesStreamRequest
-from ....chat.stream.services import ChatChoicesStreamResponse
-from ....chat.stream.services import static_check_is_chat_choices_stream_service
-from ....chat.stream.types import AiChoiceDeltas
-from ....chat.stream.types import AiChoicesDeltas
-from ....chat.stream.types import ContentAiChoiceDelta
+from ....chat.stream.types import ContentAiDelta
 from ....chat.types import ChatOption
 from ....llms.types import LlmOption
 from ....resources import UseResources
@@ -28,6 +27,10 @@ from ....stream.services import StreamResponseSink
 from ....stream.services import new_stream_response
 
 
+with lang.auto_proxy_import(globals()):
+    from .....backends.tinygrad.models import llama3 as tgl3
+
+
 ##
 
 
@@ -39,7 +42,7 @@ def _load_model(
         *,
         size: str | None = None,
         temperature: float | None = None,
-) -> tgl3.Llama3Llm:
+) -> 'tgl3.Llama3Llm':
     if size is None:
         size = DEFAULT_SIZE
     if temperature is None:
@@ -58,7 +61,7 @@ def _load_model(
 
 
 def _prepare_toks(
-        llm: tgl3.Llama3Llm,
+        llm: 'tgl3.Llama3Llm',
         chat: Chat,
         options: ta.Sequence[ChatChoicesOptions],
 ) -> list[int]:
@@ -100,7 +103,7 @@ class BaseTinygradLlama3ChatService(lang.ExitStacked, lang.Abstract):
         self._temperature = temperature
 
     @lang.cached_function(transient=True)
-    def _load_model(self) -> tgl3.Llama3Llm:
+    def _load_model(self) -> 'tgl3.Llama3Llm':
         check.not_none(self._exit_stack)
 
         return _load_model(
@@ -113,7 +116,7 @@ class BaseTinygradLlama3ChatService(lang.ExitStacked, lang.Abstract):
 
 
 # @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
-#     name='tinygrad_llama3',
+#     name='tinygrad-llama3',
 #     type='ChatChoicesService',
 # )
 @static_check_is_chat_choices_service
@@ -133,7 +136,7 @@ class TinygradLlama3ChatChoicesService(BaseTinygradLlama3ChatService):
 
 
 # @omlish-manifest $.minichain.registries.manifests.RegistryManifest(
-#     name='tinygrad_llama3',
+#     name='tinygrad-llama3',
 #     type='ChatChoicesStreamService',
 # )
 @static_check_is_chat_choices_stream_service
@@ -151,7 +154,7 @@ class TinygradLlama3ChatChoicesStreamService(BaseTinygradLlama3ChatService):
                 for s in tgl3.run_llm(llm, toks):
                     await sink.emit(AiChoicesDeltas([
                         AiChoiceDeltas([
-                            ContentAiChoiceDelta(s),
+                            ContentAiDelta(s),
                         ]),
                     ]))
 
@@ -168,5 +171,5 @@ class TinygradLlama3ChatChoicesStreamService(BaseTinygradLlama3ChatService):
 #         'ChatChoicesService',
 #         'ChatChoicesStreamService',
 #     ],
-#     'tinygrad_llama3',
+#     'tinygrad-llama3',
 # )

ommlds/minichain/backends/impls/tokenizers/tokens.py

@@ -1,23 +1,26 @@
 import typing as ta
 
-import tokenizers as tos
-
 from omlish import check
+from omlish import lang
 
 from .... import tokens as tks
 
 
+with lang.auto_proxy_import(globals()):
+    import tokenizers as tos
+
+
 ##
 
 
-def build_vocab(tos_tokenizer: tos.Tokenizer) -> tks.Vocab:
+def build_vocab(tos_tokenizer: 'tos.Tokenizer') -> tks.Vocab:
     return tks.Vocab([
         (ta.cast(tks.Token, i), tks.TokenStr(s))
         for s, i in tos_tokenizer.get_vocab().items()
     ])
 
 
-def build_specials(tos_tokenizer: tos.Tokenizer) -> tks.SpecialTokens:
+def build_specials(tos_tokenizer: 'tos.Tokenizer') -> tks.SpecialTokens:
     # FIXME
     return tks.SpecialTokens([])
 
@@ -28,7 +31,7 @@ def build_specials(tos_tokenizer: tos.Tokenizer) -> tks.SpecialTokens:
 class TokenizersTokenizer(tks.BaseTokenizer):
     def __init__(
             self,
-            tos_tokenizer: tos.Tokenizer,
+            tos_tokenizer: 'tos.Tokenizer',
     ) -> None:
         self._tos_tokenizer = check.isinstance(tos_tokenizer, tos.Tokenizer)
 
@@ -38,7 +41,7 @@ class TokenizersTokenizer(tks.BaseTokenizer):
         )
 
     @property
-    def tos_tokenizer(self) -> tos.Tokenizer:
+    def tos_tokenizer(self) -> 'tos.Tokenizer':
         return self._tos_tokenizer
 
     #

ommlds/minichain/backends/impls/transformers/sentence.py

@@ -1,7 +1,6 @@
 import typing as ta
 
-import sentence_transformers as stfm
-
+from omlish import lang
 from omlish import typedvalues as tv
 
 from ....configs import Config
@@ -13,6 +12,10 @@ from ....vectors.embeddings import static_check_is_embedding_service
 from ....vectors.types import Vector
 
 
+with lang.auto_proxy_import(globals()):
+    import sentence_transformers as stfm
+
+
 ##
 
 

ommlds/minichain/backends/impls/transformers/tokens.py

@@ -1,17 +1,20 @@
 import typing as ta
 
-import transformers as tfm
-
 from omlish import check
 from omlish import collections as col
+from omlish import lang
 
 from .... import tokens as tks
 
 
+with lang.auto_proxy_import(globals()):
+    import transformers as tfm
+
+
 ##
 
 
-def build_vocab(tfm_tokenizer: tfm.PreTrainedTokenizerBase) -> tks.Vocab:
+def build_vocab(tfm_tokenizer: 'tfm.PreTrainedTokenizerBase') -> tks.Vocab:
     return tks.Vocab([
         (ta.cast(tks.Token, i), tks.TokenStr(s))
         for s, i in tfm_tokenizer.get_vocab().items()
@@ -32,7 +35,7 @@ SPECIAL_TOKEN_ATTR_MAP: col.BiMap[type[tks.SpecialToken], str] = col.make_bi_map
 })
 
 
-def build_specials(tfm_tokenizer: tfm.PreTrainedTokenizerBase) -> tks.SpecialTokens:
+def build_specials(tfm_tokenizer: 'tfm.PreTrainedTokenizerBase') -> tks.SpecialTokens:
     return tks.SpecialTokens.from_dict({
         st: getattr(tfm_tokenizer, a)
         for st, a in SPECIAL_TOKEN_ATTR_MAP.items()
@@ -45,7 +48,7 @@ def build_specials(tfm_tokenizer: tfm.PreTrainedTokenizerBase) -> tks.SpecialTok
 class TransformersTokenizer(tks.BaseTokenizer):
     def __init__(
             self,
-            tfm_tokenizer: tfm.PreTrainedTokenizerBase,
+            tfm_tokenizer: 'tfm.PreTrainedTokenizerBase',
     ) -> None:
         self._tfm_tokenizer = check.isinstance(tfm_tokenizer, tfm.PreTrainedTokenizerBase)
 
@@ -55,7 +58,7 @@ class TransformersTokenizer(tks.BaseTokenizer):
         )
 
     @property
-    def tfm_tokenizer(self) -> tfm.PreTrainedTokenizerBase:
+    def tfm_tokenizer(self) -> 'tfm.PreTrainedTokenizerBase':
         return self._tfm_tokenizer
 
     #