omlish 0.0.0.dev217__py3-none-any.whl → 0.0.0.dev219__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev217'
-__revision__ = 'd43ca6092cff650a9fc533e97d87ce9112f6f71a'
+__version__ = '0.0.0.dev219'
+__revision__ = '5e741e0670fb75899653bd454587a1cf2add710f'
 
 
 #
@@ -42,7 +42,7 @@ class Project(ProjectBase):
         ],
 
         'compress': [
-            'lz4 ~= 4.3',
+            'lz4 ~= 4.4',
             # 'lz4 @ git+https://github.com/wrmsr/python-lz4@wrmsr_20240830_GIL_NOT_USED'
 
             'python-snappy ~= 0.7',

omlish/docker/oci/data.py

@@ -0,0 +1,71 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import dataclasses as dc
+import typing as ta
+
+from ...lite.marshal import OBJ_MARSHALER_FIELD_KEY
+from ...lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciDataclass(abc.ABC):  # noqa
+    pass
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciImageConfig(OciDataclass):
+    """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/config.md"""
+
+    architecture: str
+    os: str
+
+    @dc.dataclass(frozen=True)
+    class RootFs:
+        type: str
+        diff_ids: ta.Sequence[str]
+
+    rootfs: RootFs
+
+    #
+
+    created: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    author: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    os_version: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.version', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+    os_features: ta.Optional[ta.Sequence[str]] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.features', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+    variant: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+    """
+    config object, OPTIONAL
+        User string, OPTIONAL
+        ExposedPorts object, OPTIONAL
+        Env array of strings, OPTIONAL
+        Entrypoint array of strings, OPTIONAL
+        Cmd array of strings, OPTIONAL
+        Volumes object, OPTIONAL
+        WorkingDir string, OPTIONAL
+        Labels object, OPTIONAL
+        StopSignal string, OPTIONAL
+        ArgsEscaped boolean, OPTIONAL
+        Memory integer, OPTIONAL
+        MemorySwap integer, OPTIONAL
+        CpuShares integer, OPTIONAL
+        Healthcheck object, OPTIONAL
+    """
+    config: ta.Optional[ta.Mapping[str, ta.Any]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+    @dc.dataclass(frozen=True)
+    class History:
+        created: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        author: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        created_by: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        comment: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        empty_layer: ta.Optional[bool] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+    history: ta.Optional[ta.Sequence[History]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})

omlish/docker/oci/media.py

@@ -0,0 +1,124 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import dataclasses as dc
+import typing as ta
+
+from ...lite.check import check
+from ...lite.marshal import OBJ_MARSHALER_FIELD_KEY
+from ...lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
+from ...lite.marshal import unmarshal_obj
+from .data import OciDataclass
+from .data import OciImageConfig
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciMediaDataclass(OciDataclass, abc.ABC):  # noqa
+    SCHEMA_VERSION: ta.ClassVar[int]
+    MEDIA_TYPE: ta.ClassVar[str]
+
+
+_REGISTERED_OCI_MEDIA_DATACLASSES: ta.Dict[str, ta.Type[OciMediaDataclass]] = {}
+
+
+def _register_oci_media_dataclass(cls):
+    check.issubclass(cls, OciMediaDataclass)
+    check.arg(dc.is_dataclass(cls))
+    mt = check.non_empty_str(cls.__dict__['MEDIA_TYPE'])
+    check.not_in(mt, _REGISTERED_OCI_MEDIA_DATACLASSES)
+    _REGISTERED_OCI_MEDIA_DATACLASSES[mt] = cls
+    return cls
+
+
+def get_registered_oci_media_dataclass(media_type: str) -> ta.Optional[ta.Type[OciMediaDataclass]]:
+    return _REGISTERED_OCI_MEDIA_DATACLASSES.get(media_type)
+
+
+def unmarshal_oci_media_dataclass(
+        dct: ta.Mapping[str, ta.Any],
+        *,
+        media_type: ta.Optional[str] = None,
+) -> ta.Any:
+    if media_type is None:
+        media_type = check.non_empty_str(dct['mediaType'])
+    cls = _REGISTERED_OCI_MEDIA_DATACLASSES[media_type]
+    return unmarshal_obj(dct, cls)
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class OciMediaDescriptor(OciDataclass):
+    """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/descriptor.md#properties"""  # noqa
+
+    media_type: str = dc.field(metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})
+    digest: str
+    size: int
+
+    #
+
+    urls: ta.Optional[ta.Sequence[str]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    annotations: ta.Optional[ta.Mapping[str, str]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+    data: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    artifact_type: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'artifactType', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+
+    #
+
+    platform: ta.Optional[ta.Mapping[str, ta.Any]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+
+
+@_register_oci_media_dataclass
+@dc.dataclass(frozen=True)
+class OciMediaImageIndex(OciMediaDataclass):
+    """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/image-index.md"""
+
+    manifests: ta.Sequence[OciMediaDescriptor]  # -> OciMediaImageIndex | OciMediaImageManifest
+
+    #
+
+    annotations: ta.Optional[ta.Mapping[str, str]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+
+    #
+
+    SCHEMA_VERSION: ta.ClassVar[int] = 2
+    schema_version: int = dc.field(default=SCHEMA_VERSION, metadata={OBJ_MARSHALER_FIELD_KEY: 'schemaVersion'})
+
+    MEDIA_TYPE: ta.ClassVar[str] = 'application/vnd.oci.image.index.v1+json'
+    media_type: str = dc.field(default=MEDIA_TYPE, metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})
+
+
+@_register_oci_media_dataclass
+@dc.dataclass(frozen=True)
+class OciMediaImageManifest(OciMediaDataclass):
+    """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/manifest.md"""
+
+    config: OciMediaDescriptor  # -> OciMediaImageConfig
+
+    # MEDIA_TYPES: ta.ClassVar[ta.Mapping[str, str]] = {
+    #     'TAR': 'application/vnd.oci.image.layer.v1.tar',
+    #     'TAR_GZIP': 'application/vnd.oci.image.layer.v1.tar+gzip',
+    #     'TAR_ZSTD': 'application/vnd.oci.image.layer.v1.tar+zstd',
+    # }
+    layers: ta.Sequence[OciMediaDescriptor]
+
+    #
+
+    SCHEMA_VERSION: ta.ClassVar[int] = 2
+    schema_version: int = dc.field(default=SCHEMA_VERSION, metadata={OBJ_MARSHALER_FIELD_KEY: 'schemaVersion'})
+
+    MEDIA_TYPE: ta.ClassVar[str] = 'application/vnd.oci.image.manifest.v1+json'
+    media_type: str = dc.field(default=MEDIA_TYPE, metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})
+
+
+@_register_oci_media_dataclass
+@dc.dataclass(frozen=True)
+class OciMediaImageConfig(OciImageConfig, OciMediaDataclass):
+    SCHEMA_VERSION: ta.ClassVar[int] = 2
+    schema_version: int = dc.field(default=SCHEMA_VERSION, metadata={OBJ_MARSHALER_FIELD_KEY: 'schemaVersion'})
+
+    MEDIA_TYPE: ta.ClassVar[str] = 'application/vnd.oci.image.config.v1+json'
+    media_type: str = dc.field(default=MEDIA_TYPE, metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})

omlish/formats/json5/Json5.g4

@@ -6,9 +6,6 @@
 // See https://json5.org/
 // Derived from ../json/JSON.g4 which original derived from http://json.org
 
-// $antlr-format alignTrailingComments true, columnLimit 150, minEmptyLines 1, maxEmptyLinesToKeep 1, reflowComments false, useTab false
-// $antlr-format allowShortRulesOnASingleLine false, allowShortBlocksOnASingleLine true, alignSemicolons hanging, alignColons hanging
-
 grammar Json5;
 
 json5

omlish/http/coro/server.py

@@ -65,9 +65,12 @@ import typing as ta
 
 from ...lite.check import check
 from ...sockets.addresses import SocketAddress
-from ...sockets.handlers import SocketHandler
+from ...sockets.handlers import SocketHandler_
+from ...sockets.io import SocketIoPair
 from ..handlers import HttpHandler
 from ..handlers import HttpHandlerRequest
+from ..handlers import HttpHandlerResponseData
+from ..handlers import HttpHandlerResponseStreamedData
 from ..handlers import UnsupportedMethodHttpHandlerError
 from ..parsing import EmptyParsedHttpResult
 from ..parsing import HttpRequestParser
@@ -193,7 +196,7 @@ class CoroHttpServer:
 
         message: ta.Optional[str] = None
         headers: ta.Optional[ta.Sequence['CoroHttpServer._Header']] = None
-        data: ta.Optional[bytes] = None
+        data: ta.Optional[HttpHandlerResponseData] = None
         close_connection: ta.Optional[bool] = False
 
         def get_header(self, key: str) -> ta.Optional['CoroHttpServer._Header']:
@@ -204,7 +207,7 @@ class CoroHttpServer:
 
     #
 
-    def _build_response_bytes(self, a: _Response) -> bytes:
+    def _build_response_head_bytes(self, a: _Response) -> bytes:
         out = io.BytesIO()
 
         if a.version >= HttpProtocolVersions.HTTP_1_0:
@@ -219,11 +222,22 @@ class CoroHttpServer:
 
             out.write(b'\r\n')
 
-        if a.data is not None:
-            out.write(a.data)
-
         return out.getvalue()
 
+    def _yield_response_data(self, a: _Response) -> ta.Iterator[bytes]:
+        if a.data is None:
+            return
+
+        elif isinstance(a.data, bytes):
+            yield a.data
+            return
+
+        elif isinstance(a.data, HttpHandlerResponseStreamedData):
+            yield from a.data.iter
+
+        else:
+            raise TypeError(a.data)
+
     #
 
     DEFAULT_CONTENT_TYPE = 'text/plain'
@@ -234,8 +248,17 @@ class CoroHttpServer:
 
         if resp.get_header('Content-Type') is None:
             nh.append(self._Header('Content-Type', self._default_content_type))
+
         if resp.data is not None and resp.get_header('Content-Length') is None:
-            nh.append(self._Header('Content-Length', str(len(resp.data))))
+            cl: ta.Optional[int]
+            if isinstance(resp.data, bytes):
+                cl = len(resp.data)
+            elif isinstance(resp.data, HttpHandlerResponseStreamedData):
+                cl = resp.data.length
+            else:
+                raise TypeError(resp.data)
+            if cl is not None:
+                nh.append(self._Header('Content-Length', str(cl)))
 
         if nh:
             kw.update(headers=[*(resp.headers or []), *nh])
@@ -409,9 +432,13 @@ class CoroHttpServer:
 
                 elif isinstance(o, self._Response):
                     i = None
+
                     r = self._preprocess_response(o)
-                    b = self._build_response_bytes(r)
-                    check.none((yield self.WriteIo(b)))
+                    hb = self._build_response_head_bytes(r)
+                    check.none((yield self.WriteIo(hb)))
+
+                    for b in self._yield_response_data(r):
+                        yield self.WriteIo(b)
 
                 else:
                     raise TypeError(o)
@@ -530,27 +557,20 @@ class CoroHttpServer:
 ##
 
 
-class CoroHttpServerSocketHandler(SocketHandler):
+class CoroHttpServerSocketHandler(SocketHandler_):
     def __init__(
             self,
-            client_address: SocketAddress,
-            rfile: ta.BinaryIO,
-            wfile: ta.BinaryIO,
-            *,
             server_factory: CoroHttpServerFactory,
+            *,
             log_handler: ta.Optional[ta.Callable[[CoroHttpServer, CoroHttpServer.AnyLogIo], None]] = None,
     ) -> None:
-        super().__init__(
-            client_address,
-            rfile,
-            wfile,
-        )
+        super().__init__()
 
         self._server_factory = server_factory
         self._log_handler = log_handler
 
-    def handle(self) -> None:
-        server = self._server_factory(self._client_address)
+    def __call__(self, client_address: SocketAddress, fp: SocketIoPair) -> None:
+        server = self._server_factory(client_address)
 
         gen = server.coro_handle()
 
@@ -562,15 +582,15 @@ class CoroHttpServerSocketHandler(SocketHandler):
                     self._log_handler(server, o)
 
             elif isinstance(o, CoroHttpServer.ReadIo):
-                i = self._rfile.read(o.sz)
+                i = fp.r.read(o.sz)
 
             elif isinstance(o, CoroHttpServer.ReadLineIo):
-                i = self._rfile.readline(o.sz)
+                i = fp.r.readline(o.sz)
 
             elif isinstance(o, CoroHttpServer.WriteIo):
                 i = None
-                self._wfile.write(o.data)
-                self._wfile.flush()
+                fp.w.write(o.data)
+                fp.w.flush()
 
             else:
                 raise TypeError(o)

omlish/http/handlers.py

@@ -1,5 +1,6 @@
 # ruff: noqa: UP006 UP007
 # @omlish-lite
+import abc
 import dataclasses as dc
 import http.server
 import typing as ta
@@ -9,6 +10,10 @@ from .parsing import HttpHeaders
 
 
 HttpHandler = ta.Callable[['HttpHandlerRequest'], 'HttpHandlerResponse']  # ta.TypeAlias
+HttpHandlerResponseData = ta.Union[bytes, 'HttpHandlerResponseStreamedData']  # ta.TypeAlias  # noqa
+
+
+##
 
 
 @dc.dataclass(frozen=True)
@@ -25,13 +30,25 @@ class HttpHandlerResponse:
     status: ta.Union[http.HTTPStatus, int]
 
     headers: ta.Optional[ta.Mapping[str, str]] = None
-    data: ta.Optional[bytes] = None
+    data: ta.Optional[HttpHandlerResponseData] = None
     close_connection: ta.Optional[bool] = None
 
 
+@dc.dataclass(frozen=True)
+class HttpHandlerResponseStreamedData:
+    iter: ta.Iterable[bytes]
+    length: ta.Optional[int] = None
+
+
 class HttpHandlerError(Exception):
     pass
 
 
 class UnsupportedMethodHttpHandlerError(Exception):
     pass
+
+
+class HttpHandler_(abc.ABC):  # noqa
+    @abc.abstractmethod
+    def __call__(self, req: HttpHandlerRequest) -> HttpHandlerResponse:
+        raise NotImplementedError

omlish/http/sessions.py

@@ -9,8 +9,8 @@ import typing as ta
 import zlib
 
 from .. import lang
-from .. import secrets as sec
 from ..funcs import pairs as fpa
+from ..secrets import all as sec
 from .cookies import dump_cookie
 from .cookies import parse_cookie
 from .json import JSON_TAGGER

omlish/http/simple.py

@@ -0,0 +1,101 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import concurrent.futures as cf
+import contextlib
+import functools
+import typing as ta
+
+from ..lite.check import check
+from ..sockets.addresses import SocketAndAddress
+from ..sockets.bind import CanSocketBinder
+from ..sockets.bind import SocketBinder
+from ..sockets.server.handlers import ExecutorSocketServerHandler
+from ..sockets.server.handlers import SocketHandlerSocketServerHandler
+from ..sockets.server.handlers import SocketServerHandler
+from ..sockets.server.handlers import SocketWrappingSocketServerHandler
+from ..sockets.server.handlers import StandardSocketServerHandler
+from ..sockets.server.server import SocketServer
+from ..sockets.server.threading import ThreadingSocketServerHandler
+from .coro.server import CoroHttpServer
+from .coro.server import CoroHttpServerSocketHandler
+from .handlers import HttpHandler
+from .parsing import HttpRequestParser
+from .versions import HttpProtocolVersion
+from .versions import HttpProtocolVersions
+
+
+if ta.TYPE_CHECKING:
+    import ssl
+
+
+@contextlib.contextmanager
+def make_simple_http_server(
+        bind: CanSocketBinder,
+        handler: HttpHandler,
+        *,
+        server_version: HttpProtocolVersion = HttpProtocolVersions.HTTP_1_1,
+        ssl_context: ta.Optional['ssl.SSLContext'] = None,
+        executor: ta.Optional[cf.Executor] = None,
+        use_threads: bool = False,
+) -> ta.Iterator[SocketServer]:
+    check.arg(not (executor is not None and use_threads))
+
+    #
+
+    with contextlib.ExitStack() as es:
+        server_factory = functools.partial(
+            CoroHttpServer,
+            handler=handler,
+            parser=HttpRequestParser(
+                server_version=server_version,
+            ),
+        )
+
+        socket_handler = CoroHttpServerSocketHandler(
+            server_factory,
+        )
+
+        #
+
+        server_handler: SocketServerHandler = SocketHandlerSocketServerHandler(
+            socket_handler,
+        )
+
+        #
+
+        if ssl_context is not None:
+            server_handler = SocketWrappingSocketServerHandler(
+                server_handler,
+                SocketAndAddress.socket_wrapper(functools.partial(
+                    ssl_context.wrap_socket,
+                    server_side=True,
+                )),
+            )
+
+        #
+
+        server_handler = StandardSocketServerHandler(
+            server_handler,
+        )
+
+        #
+
+        if executor is not None:
+            server_handler = ExecutorSocketServerHandler(
+                server_handler,
+                executor,
+            )
+
+        elif use_threads:
+            server_handler = es.enter_context(ThreadingSocketServerHandler(
+                server_handler,
+            ))
+
+        #
+
+        server = es.enter_context(SocketServer(
+            SocketBinder.of(bind),
+            server_handler,
+        ))
+
+        yield server

omlish/iterators/tools.py

@@ -45,6 +45,7 @@ def take(n: int, iterable: ta.Iterable[T]) -> list[T]:
 
 
 def chunk(n: int, iterable: ta.Iterable[T], strict: bool = False) -> ta.Iterator[list[T]]:
+    # TODO: remove with 3.13 - 3.12 doesn't support strict
     iterator = iter(functools.partial(take, n, iter(iterable)), [])
     if strict:
         def ret():

omlish/logs/all.py

@@ -1,3 +1,7 @@
+from .callers import (  # noqa
+    LoggingCaller,
+)
+
 from .color import (  # noqa
     ColorLogFormatter,
 )
@@ -18,6 +22,15 @@ from .noisy import (  # noqa
     silence_noisy_loggers,
 )
 
+from .protocol import (  # noqa
+    LogLevel,
+
+    Logging,
+    NopLogging,
+    AbstractLogging,
+    StdlibLogging,
+)
+
 from .proxy import (  # noqa
     ProxyLogFilterer,
     ProxyLogHandler,

omlish/logs/callers.py

@@ -0,0 +1,45 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import io
+import sys
+import traceback
+import types
+import typing as ta
+
+
+class LoggingCaller(ta.NamedTuple):
+    filename: str
+    lineno: int
+    func: str
+    sinfo: ta.Optional[str]
+
+    @classmethod
+    def find_frame(cls, ofs: int = 0) -> types.FrameType:
+        f: ta.Any = sys._getframe(2 + ofs)  # noqa
+        while hasattr(f, 'f_code'):
+            if f.f_code.co_filename != __file__:
+                return f
+            f = f.f_back
+        raise RuntimeError
+
+    @classmethod
+    def find(cls, stack_info: bool = False) -> 'LoggingCaller':
+        f = cls.find_frame(1)
+        # TODO: ('(unknown file)', 0, '(unknown function)', None) ?
+
+        sinfo = None
+        if stack_info:
+            sio = io.StringIO()
+            sio.write('Stack (most recent call last):\n')
+            traceback.print_stack(f, file=sio)
+            sinfo = sio.getvalue()
+            sio.close()
+            if sinfo[-1] == '\n':
+                sinfo = sinfo[:-1]
+
+        return cls(
+            f.f_code.co_filename,
+            f.f_lineno,
+            f.f_code.co_name,
+            sinfo,
+        )