ommlds 0.0.0.dev466__py3-none-any.whl → 0.0.0.dev512__py3-none-any.whl

ommlds/minichain/content/transform/strings.py

@@ -0,0 +1,33 @@
+import typing as ta
+
+from omlish import dataclasses as dc
+
+from ..content import Content
+from ..emphasis import EmphasisContent
+from ..metadata import ContentOriginal
+from ..text import TextContent
+from .visitors import VisitorContentTransform
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class StringFnContentTransform(VisitorContentTransform[None]):
+    fn: ta.Callable[[str], str]
+
+    def visit_str(self, c: str, ctx: None) -> TextContent:
+        return TextContent(self.fn(c)).with_metadata(ContentOriginal(c))
+
+    def visit_text_content(self, c: TextContent, ctx: None) -> TextContent:
+        return c.replace(s=self.fn(c.s))
+
+    def visit_emphasis_content(self, c: EmphasisContent, ctx: None) -> EmphasisContent:
+        return c.replace(s=self.fn(c.s))
+
+
+def transform_content_strings(fn: ta.Callable[[str], str], o: Content) -> Content:
+    return StringFnContentTransform(fn).transform(o, None)

ommlds/minichain/content/transform/templates.py

@@ -0,0 +1,25 @@
+from omlish import check
+from omlish.text import templating as tpl
+
+from ..content import Content
+from ..metadata import ContentOriginal
+from ..templates import TemplateContent
+from ..text import TextContent
+from .visitors import VisitorContentTransform
+
+
+##
+
+
+class TemplateContentMaterializer(VisitorContentTransform[None]):
+    def __init__(
+            self,
+            templater_context: tpl.Templater.Context | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._templater_context = templater_context
+
+    def visit_template_content(self, c: TemplateContent, ctx: None) -> Content:
+        s = c.t.render(check.not_none(self._templater_context))
+        return TextContent(s).with_metadata(ContentOriginal(c))

ommlds/minichain/content/transform/types.py

@@ -0,0 +1,18 @@
+import abc
+import typing as ta
+
+from omlish import lang
+
+from ..content import Content
+
+
+C = ta.TypeVar('C')
+
+
+##
+
+
+class ContentTransform(lang.Abstract, ta.Generic[C]):
+    @abc.abstractmethod
+    def transform(self, c: Content, ctx: C) -> Content:
+        raise NotImplementedError

ommlds/minichain/content/transform/visitors.py

@@ -0,0 +1,38 @@
+import collections.abc
+import typing as ta
+
+from omlish import check
+from omlish import lang
+
+from ..composite import CompositeContent
+from ..content import Content
+from ..visitors import ContentVisitor
+from .types import ContentTransform
+
+
+C = ta.TypeVar('C')
+
+
+##
+
+
+class VisitorContentTransform(ContentVisitor[C, Content], ContentTransform[C], lang.Abstract):
+    @ta.final
+    def transform(self, c: Content, ctx: C) -> Content:
+        """Final - must be identical to `visit`."""
+
+        return self.visit(c, ctx)
+
+    #
+
+    def visit_content(self, c: Content, ctx: C) -> Content:
+        return c
+
+    def visit_sequence(self, c: ta.Sequence[Content], ctx: C) -> Content:
+        return lang.map_preserve(lambda cc: self.visit(cc, ctx), c)
+
+    def visit_composite_content(self, c: CompositeContent, ctx: C) -> Content:
+        cc = c.child_content()
+        ncc = self.visit_sequence(cc, ctx)
+        nc = c.replace_child_content(check.isinstance(ncc, collections.abc.Sequence))
+        return super().visit_composite_content(nc, ctx)

ommlds/minichain/content/visitors.py

@@ -0,0 +1,218 @@
+import collections.abc
+import inspect
+import typing as ta
+
+from omlish import collections as col
+from omlish import lang
+
+from .code import BlockCodeContent
+from .code import CodeContent
+from .code import InlineCodeContent
+from .composite import CompositeContent
+from .content import BaseContent
+from .content import Content
+from .dynamic import DynamicContent
+from .emphasis import BoldContent
+from .emphasis import BoldItalicContent
+from .emphasis import EmphasisContent
+from .emphasis import ItalicContent
+from .images import ImageContent
+from .json import JsonContent
+from .link import LinkContent
+from .markdown import MarkdownContent
+from .namespaces import NamespaceContent
+from .placeholders import PlaceholderContent
+from .quote import QuoteContent
+from .recursive import RecursiveContent
+from .resources import ResourceContent
+from .section import SectionContent
+from .sequence import BlockContent
+from .sequence import InlineContent
+from .sequence import ItemListContent
+from .sequence import SequenceContent
+from .standard import StandardContent
+from .tag import TagContent
+from .templates import TemplateContent
+from .text import TextContent
+
+
+C = ta.TypeVar('C')
+R = ta.TypeVar('R')
+
+
+##
+
+
+class ContentVisitor(lang.Abstract, ta.Generic[C, R]):
+    _visit_method_map: ta.ClassVar[ta.Mapping[ta.Any, str]]
+
+    def visit(self, c: Content, ctx: C) -> R:
+        if isinstance(c, str):
+            return self.visit_str(c, ctx)
+
+        if isinstance(c, collections.abc.Sequence):
+            return self.visit_sequence(c, ctx)
+
+        try:
+            a = self._visit_method_map[type(c)]
+        except KeyError:
+            raise TypeError(c) from None
+
+        return getattr(self, a)(c, ctx)
+
+    ##
+    # per-type visit methods
+
+    def visit_content(self, c: Content, ctx: C) -> R:
+        raise TypeError(c)
+
+    ##
+    # non-BaseContent
+
+    def visit_str(self, c: str, ctx: C) -> R:
+        return self.visit_content(c, ctx)
+
+    def visit_sequence(self, c: ta.Sequence[Content], ctx: C) -> R:
+        return self.visit_content(c, ctx)
+
+    ##
+    # BaseContent
+
+    def visit_base_content(self, c: BaseContent, ctx: C) -> R:
+        return self.visit_content(c, ctx)
+
+    ##
+    # StandardContent
+
+    def visit_standard_content(self, c: StandardContent, ctx: C) -> R:
+        return self.visit_base_content(c, ctx)
+
+    ##
+    # leaf StandardContent
+
+    def visit_image_content(self, c: ImageContent, ctx: C) -> R:
+        return self.visit_standard_content(c, ctx)
+
+    def visit_json_content(self, c: JsonContent, ctx: C) -> R:
+        return self.visit_standard_content(c, ctx)
+
+    def visit_link_content(self, c: LinkContent, ctx: C) -> R:
+        return self.visit_standard_content(c, ctx)
+
+    def visit_markdown_content(self, c: MarkdownContent, ctx: C) -> R:
+        return self.visit_standard_content(c, ctx)
+
+    def visit_quote_content(self, c: QuoteContent, ctx: C) -> R:
+        return self.visit_standard_content(c, ctx)
+
+    def visit_text_content(self, c: TextContent, ctx: C) -> R:
+        return self.visit_standard_content(c, ctx)
+
+    ##
+    # CompositeContent
+
+    def visit_composite_content(self, c: CompositeContent, ctx: C) -> R:
+        return self.visit_standard_content(c, ctx)
+
+    def visit_section_content(self, c: SectionContent, ctx: C) -> R:
+        return self.visit_composite_content(c, ctx)
+
+    def visit_tag_content(self, c: TagContent, ctx: C) -> R:
+        return self.visit_composite_content(c, ctx)
+
+    ##
+    # CodeContent
+
+    def visit_code_content(self, c: CodeContent, ctx: C) -> R:
+        return self.visit_standard_content(c, ctx)
+
+    def visit_inline_code_content(self, c: InlineCodeContent, ctx: C) -> R:
+        return self.visit_code_content(c, ctx)
+
+    def visit_block_code_content(self, c: BlockCodeContent, ctx: C) -> R:
+        return self.visit_code_content(c, ctx)
+
+    ##
+    # DynamicContent
+
+    def visit_dynamic_content(self, c: DynamicContent, ctx: C) -> R:
+        return self.visit_standard_content(c, ctx)
+
+    def visit_recursive_content(self, c: RecursiveContent, ctx: C) -> R:
+        return self.visit_dynamic_content(c, ctx)
+
+    def visit_namespace_content(self, c: NamespaceContent, ctx: C) -> R:
+        return self.visit_recursive_content(c, ctx)
+
+    def visit_placeholder_content(self, c: PlaceholderContent, ctx: C) -> R:
+        return self.visit_recursive_content(c, ctx)
+
+    def visit_resource_content(self, c: ResourceContent, ctx: C) -> R:
+        return self.visit_recursive_content(c, ctx)
+
+    def visit_template_content(self, c: TemplateContent, ctx: C) -> R:
+        return self.visit_dynamic_content(c, ctx)
+
+    ##
+    # EmphasisContent
+
+    def visit_emphasis_content(self, c: EmphasisContent, ctx: C) -> R:
+        return self.visit_standard_content(c, ctx)
+
+    def visit_bold_content(self, c: BoldContent, ctx: C) -> R:
+        return self.visit_emphasis_content(c, ctx)
+
+    def visit_italic_content(self, c: ItalicContent, ctx: C) -> R:
+        return self.visit_emphasis_content(c, ctx)
+
+    def visit_bold_italic_content(self, c: BoldItalicContent, ctx: C) -> R:
+        return self.visit_emphasis_content(c, ctx)
+
+    ##
+    # SequenceContent
+
+    def visit_sequence_content(self, c: SequenceContent, ctx: C) -> R:
+        return self.visit_composite_content(c, ctx)
+
+    def visit_inline_content(self, c: InlineContent, ctx: C) -> R:
+        return self.visit_sequence_content(c, ctx)
+
+    def visit_block_content(self, c: BlockContent, ctx: C) -> R:
+        return self.visit_sequence_content(c, ctx)
+
+    def visit_item_list_content(self, c: ItemListContent, ctx: C) -> R:
+        return self.visit_sequence_content(c, ctx)
+
+
+ContentVisitor._visit_method_map = col.make_map([  # noqa
+    (list(inspect.signature(o).parameters.values())[1].annotation, a)
+    for a, o in ContentVisitor.__dict__.items()
+    if a.startswith('visit_')
+], strict=True)
+
+
+##
+
+
+class StandardContentVisitorTypeError(TypeError):
+    pass
+
+
+class StandardContentVisitor(ContentVisitor[C, R], lang.Abstract):
+    def visit_str(self, c: str, ctx: C) -> R:
+        raise StandardContentVisitorTypeError(c)
+
+    def visit_sequence(self, c: ta.Sequence[Content], ctx: C) -> R:
+        raise StandardContentVisitorTypeError(c)
+
+
+##
+
+
+class StaticContentVisitorTypeError(TypeError):
+    pass
+
+
+class StaticContentVisitor(ContentVisitor[C, R], lang.Abstract):
+    def visit_dynamic_content(self, c: DynamicContent, ctx: C) -> R:
+        raise StaticContentVisitorTypeError(c)

ommlds/minichain/http/stream.py

@@ -0,0 +1,195 @@
+"""
+TODO:
+ - better pipeline composition lol
+"""
+import typing as ta
+
+from omlish import check
+from omlish import dataclasses as dc
+from omlish import lang
+from omlish.http import all as http
+from omlish.http import sse
+from omlish.io.buffers import DelimitingBuffer
+
+from ..resources import UseResources
+from ..stream.services import StreamResponse
+from ..stream.services import StreamResponseSink
+from ..stream.services import new_stream_response
+from ..types import Option
+from ..types import Output
+
+
+##
+
+
+@dc.dataclass()
+@dc.extra_class_params(default_repr_fn=lang.opt_repr)
+class HttpStreamResponseError(Exception):
+    response: http.BaseHttpResponse
+
+    data: bytes | None = None
+    data_exception: Exception | None = None
+
+    @classmethod
+    async def from_response(cls, response: http.AsyncStreamHttpResponse) -> 'HttpStreamResponseError':
+        data: bytes | None = None
+        data_exception: Exception | None = None
+
+        try:
+            data = await response.stream.readall()
+        except Exception as de:  # noqa
+            data_exception = de
+
+        return HttpStreamResponseError(
+            response,
+            data=data,
+            data_exception=data_exception,
+        )
+
+
+##
+
+
+class HttpStreamResponseHandler(lang.Abstract):
+    def start(self) -> ta.Sequence[Output]:
+        return ()
+
+    def finish(self) -> ta.Sequence[Output]:
+        return ()
+
+
+##
+
+
+class BytesHttpStreamResponseHandler(HttpStreamResponseHandler, lang.Abstract):
+    def process_bytes(self, data: bytes) -> ta.Iterable:
+        return ()
+
+
+class BytesHttpStreamResponseBuilder:
+    def __init__(
+            self,
+            http_client: http.AsyncHttpClient | None,
+            handling: ta.Callable[[http.AsyncStreamHttpResponse], BytesHttpStreamResponseHandler],
+            *,
+            read_chunk_size: int = -1,
+    ) -> None:
+        super().__init__()
+
+        self._http_client = http_client
+        self._handling = handling
+        self._read_chunk_size = read_chunk_size
+
+    async def new_stream_response(
+            self,
+            http_request: http.HttpRequest,
+            options: ta.Sequence[Option],
+    ) -> StreamResponse:
+        async with UseResources.or_new(options) as rs:
+            http_client = await rs.enter_async_context(http.manage_async_client(self._http_client))
+            http_response = await rs.enter_async_context(await http_client.stream_request(http_request))
+
+            if http_response.status != 200:
+                raise await HttpStreamResponseError.from_response(http_response)
+
+            handler = self._handling(http_response)
+
+            async def inner(sink: StreamResponseSink) -> ta.Sequence | None:
+                while True:
+                    b = await http_response.stream.read1(self._read_chunk_size)
+
+                    for v in handler.process_bytes(b):
+                        if v is None:
+                            break
+
+                        await sink.emit(v)
+
+                    if not b:
+                        break
+
+                return handler.finish()
+
+            return await new_stream_response(
+                rs,
+                inner,
+                handler.start(),
+            )
+
+
+##
+
+
+class LinesHttpStreamResponseHandler(HttpStreamResponseHandler, lang.Abstract):
+    def process_line(self, line: bytes) -> ta.Iterable:
+        return ()
+
+    def as_bytes(self) -> BytesHttpStreamResponseHandler:
+        return LinesBytesHttpStreamResponseHandler(self)
+
+
+class LinesBytesHttpStreamResponseHandler(BytesHttpStreamResponseHandler):
+    def __init__(self, handler: LinesHttpStreamResponseHandler) -> None:
+        super().__init__()
+
+        self._handler = handler
+
+        self._db = DelimitingBuffer([b'\r', b'\n', b'\r\n'])
+
+    def start(self) -> ta.Sequence[Output]:
+        return self._handler.start()
+
+    def process_bytes(self, data: bytes) -> ta.Iterable:
+        for o in self._db.feed(data):
+            if isinstance(o, bytes):
+                yield from self._handler.process_line(o)
+
+            else:
+                raise TypeError(o)
+
+    def finish(self) -> ta.Sequence[Output]:
+        check.state(self._db.is_closed)
+
+        return self._handler.finish()
+
+
+##
+
+
+class SseHttpStreamResponseHandler(HttpStreamResponseHandler, lang.Abstract):
+    def process_sse(self, so: sse.SseDecoderOutput) -> ta.Iterable:
+        return ()
+
+    def as_lines(self) -> LinesHttpStreamResponseHandler:
+        return SseLinesHttpStreamResponseHandler(self)
+
+
+class SseLinesHttpStreamResponseHandler(LinesHttpStreamResponseHandler):
+    def __init__(self, handler: SseHttpStreamResponseHandler) -> None:
+        super().__init__()
+
+        self._handler = handler
+
+        self._sd = sse.SseDecoder()
+
+    def start(self) -> ta.Sequence[Output]:
+        return self._handler.start()
+
+    def process_line(self, line: bytes) -> ta.Iterable:
+        for so in self._sd.process_line(line):
+            yield from self._handler.process_sse(so)
+
+    def finish(self) -> ta.Sequence[Output]:
+        return self._handler.finish()
+
+
+#
+
+
+class SimpleSseLinesHttpStreamResponseHandler(SseHttpStreamResponseHandler):
+    def __init__(self, fn: ta.Callable[[sse.SseDecoderOutput], ta.Iterable]) -> None:
+        super().__init__()
+
+        self._fn = fn
+
+    def process_sse(self, so: sse.SseDecoderOutput) -> ta.Iterable:
+        return self._fn(so)

ommlds/minichain/lib/fs/tools/read.py

@@ -27,7 +27,7 @@ MAX_LINE_LENGTH = 2_000
 
 @tool_spec_override(
     desc=f"""
-         Reads a file from the local filesystem. You can access any file directly by using this tool.
+        Reads a file from the local filesystem. You can access any file directly by using this tool.
 
         Assume this tool is able to read all files on the machine. If the User provides a path to a file assume that
         path is valid. It is okay to read a file that does not exist; an error will be returned.

ommlds/minichain/lib/fs/tools/recursivels/rendering.py

@@ -5,10 +5,10 @@ TODO:
   - any directory with unlisted children will be suffixed inline with '...'
   - absence of '...' indicates an empty directory
 """
-import dataclasses as dc
 import typing as ta
 
 from omlish import check
+from omlish import dataclasses as dc
 from omlish import lang
 
 from .running import DirLsItem

ommlds/minichain/lib/fs/tools/recursivels/running.py

@@ -1,8 +1,8 @@
-import dataclasses as dc
 import os.path
 import typing as ta
 
 from omlish import check
+from omlish import dataclasses as dc
 from omlish import lang
 
 

ommlds/minichain/lib/todo/tools/write.py

@@ -8,6 +8,7 @@ from omlish import lang
 from omlish import marshal as msh
 
 from ....content.namespaces import ContentNamespace
+from ....content.namespaces import NamespaceContent
 from ....tools.execution.catalog import ToolCatalogEntry
 from ....tools.execution.reflect import reflect_tool_catalog_entry
 from ....tools.reflect import tool_spec_override
@@ -311,7 +312,7 @@ class TodoWriteDescriptionChunks(ContentNamespace):
 
 
 @tool_spec_override(
-    desc=TodoWriteDescriptionChunks,
+    desc=NamespaceContent(TodoWriteDescriptionChunks),
 )
 def execute_todo_write_tool(todo_items: ta.Sequence[TodoItem]) -> ta.Sequence[TodoItem]:
     if todo_items:

ommlds/minichain/lib/todo/types.py

@@ -3,9 +3,9 @@ TODO:
  - enums lol
   - * with ToolParam-like desc metadata somehow *
 """
-import dataclasses as dc
 import typing as ta
 
+from omlish import dataclasses as dc
 from omlish import lang
 
 from ...tools.reflect import tool_param_metadata

ommlds/minichain/llms/types.py

@@ -31,6 +31,10 @@ class MaxTokens(LlmOption, tv.UniqueScalarTypedValue[int]):
     pass
 
 
+class MaxCompletionTokens(LlmOption, tv.UniqueScalarTypedValue[int]):
+    pass
+
+
 ##
 
 

ommlds/minichain/metadata.py

@@ -3,9 +3,13 @@ import datetime
 import typing as ta
 import uuid
 
+from omlish import check
+from omlish import dataclasses as dc
 from omlish import lang
 from omlish import typedvalues as tv
 
+from ._typedvalues import _tv_field_metadata
+
 
 ##
 
@@ -21,9 +25,9 @@ MetadataT = ta.TypeVar('MetadataT', bound=Metadata)
 
 
 class MetadataContainer(
-    tv.TypedValueGeneric[MetadataT],
     lang.Abstract,
     lang.PackageSealed,
+    ta.Generic[MetadataT],
 ):
     @property
     @abc.abstractmethod
@@ -31,13 +35,63 @@ class MetadataContainer(
         raise NotImplementedError
 
     @abc.abstractmethod
-    def with_metadata(self, *mds: MetadataT, override: bool = False) -> ta.Self:
+    def with_metadata(
+            self,
+            *add: MetadataT,
+            discard: ta.Iterable[type] | None = None,
+            override: bool = False,
+    ) -> ta.Self:
         raise NotImplementedError
 
 
 ##
 
 
+@dc.dataclass(frozen=True)
+class MetadataContainerDataclass(  # noqa
+    MetadataContainer[MetadataT],
+    lang.Abstract,
+):
+    def __init_subclass__(cls, **kwargs: ta.Any) -> None:
+        super().__init_subclass__(**kwargs)
+
+        check.state(hasattr(cls, '_metadata'))
+
+    @staticmethod
+    def _configure_metadata_field(md_fld, md_cls):
+        dc.set_field_metadata(
+            check.isinstance(md_fld, dc.Field),
+            _tv_field_metadata(
+                check.not_none(md_cls),
+                marshal_name='metadata',
+            ),
+        )
+
+    @property
+    def metadata(self) -> tv.TypedValues[MetadataT]:
+        return check.isinstance(getattr(self, '_metadata'), tv.TypedValues)
+
+    def with_metadata(
+            self,
+            *add: MetadataT,
+            discard: ta.Iterable[type] | None = None,
+            override: bool = False,
+    ) -> ta.Self:
+        new = (old := self.metadata).update(
+            *add,
+            discard=discard,
+            override=override,
+        )
+
+        if new is old:
+            return self
+
+        return dc.replace(self, _metadata=new)  # type: ignore[call-arg]  # noqa
+
+
+##
+
+
 class CommonMetadata(Metadata, lang.Abstract):
     pass