omlish 0.0.0.dev39__py3-none-any.whl → 0.0.0.dev41__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev39'
-__revision__ = 'a18b6ac9698db2d5d932d6055a4828fd9f6c5848-dirty'
+__version__ = '0.0.0.dev41'
+__revision__ = '9666ba3a1c40b95074dfcacd1c0c87ca5e267d6f'
 
 
 #
@@ -40,7 +40,8 @@ class Project(ProjectBase):
         ],
 
         'compress': [
-            'lz4 ~= 4.0',
+            'lz4 ~= 4.3',
+            # 'lz4 @ git+https://github.com/wrmsr/python-lz4@wrmsr_20240830_GIL_NOT_USED'
 
             'python-snappy ~= 0.7; python_version < "3.13"',
 
@@ -83,7 +84,7 @@ class Project(ProjectBase):
             'sqlalchemy[asyncio] ~= 2.0',
         ],
 
-        'sqlalchemy-drivers': [
+        'sqldrivers': [
             'pg8000 ~= 1.31',
             # 'psycopg2 ~= 2.9',
             # 'psycopg ~= 3.2',
@@ -96,6 +97,8 @@ class Project(ProjectBase):
             'aiosqlite ~= 0.20',
             'asyncpg ~= 0.29; python_version < "3.13"',
 
+            'apsw ~= 3.46',
+
             'sqlean.py ~= 3.45; python_version < "3.13"',
 
             'duckdb ~= 1.1',

omlish/argparse.py

@@ -73,7 +73,7 @@ class Command:
 
     def __post_init__(self) -> None:
         check.isinstance(self.name, str)
-        check.not_in('-', self.name)
+        check.not_in('_', self.name)
         check.not_empty(self.name)
 
         check.callable(self.fn)
@@ -103,7 +103,7 @@ def command(
 
     def inner(fn):
         return Command(
-            (name if name is not None else fn.__name__).replace('-', '_'),
+            (name if name is not None else fn.__name__).replace('_', '-'),
             fn,
             args,
             parent=parent,

omlish/bootstrap/base.py

@@ -2,7 +2,7 @@ import abc
 import dataclasses as dc
 import typing as ta
 
-from omlish import lang
+from .. import lang
 
 
 BootstrapConfigT = ta.TypeVar('BootstrapConfigT', bound='Bootstrap.Config')

omlish/c3.py

@@ -43,7 +43,8 @@ T = ta.TypeVar('T')
 
 
 def merge(seqs: ta.MutableSequence[list[T]]) -> list[T]:
-    """Merges MROs in *sequences* to a single MRO using the C3 algorithm.
+    """
+    Merges MROs in *sequences* to a single MRO using the C3 algorithm.
 
     Adapted from https://www.python.org/download/releases/2.3/mro/.
     """
@@ -78,7 +79,8 @@ def mro(
         get_bases: ta.Callable[[T], ta.Sequence[T]] = operator.attrgetter('__bases__'),
         is_subclass: ta.Callable[[T, T], bool] = issubclass,  # type: ignore
 ) -> list[T]:
-    """Computes the method resolution order using extended C3 linearization.
+    """
+    Computes the method resolution order using extended C3 linearization.
 
     If no *abcs* are given, the algorithm works exactly like the built-in C3 linearization used for method resolution.
 
@@ -128,7 +130,8 @@ def compose_mro(
         is_subclass: ta.Callable[[T, T], bool] = issubclass,  # type: ignore
         get_subclasses: ta.Callable[[T], ta.Iterable[T]] = operator.methodcaller('__subclasses__'),
 ) -> list[T]:
-    """Calculates the method resolution order for a given class *cls*.
+    """
+    Calculates the method resolution order for a given class *cls*.
 
     Includes relevant abstract base classes (with their respective bases) from the *types* list. Uses a modified C3
     linearization algorithm.

omlish/check.py

@@ -44,41 +44,49 @@ def unregister_on_raise(fn: OnRaiseFn) -> None:
 #
 
 
-_render_args: ta.Callable[..., str | None] | None = None
+_ARGS_RENDERER: ta.Callable[..., str | None] | None = None
 
 
-def enable_args_rendering() -> bool:
-    global _render_args
-    if _render_args is not None:
+def _try_enable_args_rendering() -> bool:
+    global _ARGS_RENDERER
+    if _ARGS_RENDERER is not None:
         return True
 
-    with _CONFIG_LOCK:
-        if _render_args is not None:
-            return True  # type: ignore
+    try:
+        from .diag.asts import ArgsRenderer
 
-        try:
-            from .diag.asts import ArgsRenderer
+        ArgsRenderer.smoketest()
 
-            ArgsRenderer.smoketest()
+    except Exception:  # noqa
+        return False
 
-        except Exception:  # noqa
-            return False
+    def _real_render_args(fmt: str, *args: ta.Any) -> str | None:
+        ra = ArgsRenderer(back=3).render_args(*args)
+        if ra is None:
+            return None
 
-        def _real_render_args(fmt: str, *args: ta.Any) -> str | None:
-            ra = ArgsRenderer(back=3).render_args(*args)
-            if ra is None:
-                return None
+        return fmt % tuple(str(a) for a in ra)
 
-            return fmt % tuple(str(a) for a in ra)
+    _ARGS_RENDERER = _real_render_args
+    return True
 
-        _render_args = _real_render_args
-        return True
 
+_TRIED_ENABLED_ARGS_RENDERING: bool | None = None
 
-enable_args_rendering()
 
+def try_enable_args_rendering() -> bool:
+    global _TRIED_ENABLED_ARGS_RENDERING
+    if _TRIED_ENABLED_ARGS_RENDERING is not None:
+        return _TRIED_ENABLED_ARGS_RENDERING
 
-#
+    with _CONFIG_LOCK:
+        if _TRIED_ENABLED_ARGS_RENDERING is None:
+            _TRIED_ENABLED_ARGS_RENDERING = _try_enable_args_rendering()
+
+        return _TRIED_ENABLED_ARGS_RENDERING
+
+
+##
 
 
 def _default_exception_factory(exc_cls: type[Exception], *args, **kwargs) -> Exception:
@@ -111,8 +119,8 @@ def _raise(
     if message is None:
         message = default_message
 
-    if render_fmt is not None and _render_args is not None:
-        rendered_args = _render_args(render_fmt, *ak.args)
+    if render_fmt is not None and _ARGS_RENDERER is not None:
+        rendered_args = _ARGS_RENDERER(render_fmt, *ak.args)
         if rendered_args is not None:
             message = f'{message} : {rendered_args}'
 
@@ -124,6 +132,8 @@ def _raise(
         **ak.kwargs,
     )
 
+    try_enable_args_rendering()
+
     for fn in _ON_RAISE:
         fn(exc)
 

omlish/concurrent/threadlets.py

@@ -7,7 +7,7 @@ import abc
 import dataclasses as dc
 import typing as ta
 
-from omlish import lang
+from .. import lang
 
 
 if ta.TYPE_CHECKING:

omlish/diag/pycharm.py

@@ -9,9 +9,9 @@ from .. import lang
 
 
 if ta.TYPE_CHECKING:
-    docker = lang.proxy_import('omlish.docker')
+    docker = lang.proxy_import('..docker')
 else:
-    from omlish import docker
+    from .. import docker
 
 
 ##

omlish/docker/__init__.py

@@ -0,0 +1,27 @@
+from .cli import (  # noqa
+    Inspect,
+    Port,
+    PsItem,
+    cli_inspect,
+    cli_ps,
+    has_cli,
+    parse_port,
+)
+
+from .compose import (  # noqa
+    ComposeConfig,
+    get_compose_port,
+)
+
+from .helpers import (  # noqa
+    DOCKER_FOR_MAC_HOSTNAME,
+    DOCKER_HOST_PLATFORM_KEY,
+    get_docker_host_platform,
+    is_likely_in_docker,
+    timebomb_payload,
+)
+
+from .hub import (  # noqa
+    HubRepoInfo,
+    get_hub_repo_info,
+)

omlish/docker/cli.py

@@ -0,0 +1,101 @@
+"""
+TODO:
+ - https://github.com/mag37/dockcheck/blob/3d122f2b868eb53a25a3014f0f6bd499390a3a29/dockcheck.sh
+  - https://github.com/regclient/regclient
+  - https://stackoverflow.com/questions/71409458/how-to-download-docker-image-using-http-api-using-docker-hub-credentials
+  - https://stackoverflow.com/questions/55386202/how-can-i-use-the-docker-registry-api-to-pull-information-about-a-container-get
+  - https://ops.tips/blog/inspecting-docker-image-without-pull/
+"""  # noqa
+import datetime
+import re
+import subprocess
+import typing as ta
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+from .. import marshal as msh
+from ..formats import json
+
+
+@dc.dataclass(frozen=True)
+@msh.update_object_metadata(field_naming=msh.Naming.CAMEL, unknown_field='x')
+@msh.update_fields_metadata(['id'], name='ID')
+class PsItem(lang.Final):
+    command: str
+    created_at: datetime.datetime
+    id: str
+    image: str
+    labels: str
+    local_volumes: str
+    mounts: str
+    names: str
+    networks: str
+    ports: str
+    running_for: str
+    size: str
+    state: str
+    status: str
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+class Port(ta.NamedTuple):
+    ip: str
+    from_port: int
+    to_port: int
+    proto: str
+
+
+_PORT_PAT = re.compile(r'(?P<ip>[^:]+):(?P<from_port>\d+)->(?P<to_port>\d+)/(?P<proto>\w+)')
+
+
+def parse_port(s: str) -> Port:
+    # '0.0.0.0:35221->22/tcp, 0.0.0.0:35220->8000/tcp'
+    m = check.not_none(_PORT_PAT.fullmatch(s))
+    return Port(
+        m.group('ip'),
+        int(m.group('from_port')),
+        int(m.group('to_port')),
+        m.group('proto'),
+    )
+
+
+def cli_ps() -> list[PsItem]:
+    o = subprocess.check_output([
+        'docker',
+        'ps',
+        '--no-trunc',
+        '--format', '{{json .}}',
+    ])
+
+    ret: list[PsItem] = []
+    for l in o.decode().splitlines():
+        d = json.loads(l)
+        pi = msh.unmarshal(d, PsItem)
+        ret.append(pi)
+
+    return ret
+
+
+@dc.dataclass(frozen=True)
+@msh.update_object_metadata(field_naming=msh.Naming.CAMEL, unknown_field='x')
+class Inspect(lang.Final):
+    id: str
+    created: datetime.datetime
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+def cli_inspect(ids: list[str]) -> list[Inspect]:
+    o = subprocess.check_output(['docker', 'inspect', *ids])
+    return msh.unmarshal(json.loads(o.decode()), list[Inspect])
+
+
+def has_cli() -> bool:
+    try:
+        proc = subprocess.run(['docker', '--version'])  # noqa
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        return False
+    else:
+        return not proc.returncode

omlish/docker/compose.py

@@ -0,0 +1,51 @@
+"""
+TODO:
+ - merged compose configs: https://github.com/wrmsr/bane/blob/27647abdcfb323b73e6982a5c318c7029496b203/core/dev/docker/compose.go#L38
+"""  # noqa
+import typing as ta
+
+from .. import check
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    import yaml
+else:
+    yaml = lang.proxy_import('yaml')
+
+
+class ComposeConfig:
+    def __init__(
+            self,
+            prefix: str,
+            *,
+            file_path: str | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._prefix = prefix
+        self._file_path = file_path
+
+    @lang.cached_function
+    def get_config(self) -> ta.Mapping[str, ta.Any]:
+        with open(check.not_none(self._file_path)) as f:
+            buf = f.read()
+        return yaml.safe_load(buf)
+
+    @lang.cached_function
+    def get_services(self) -> ta.Mapping[str, ta.Any]:
+        ret = {}
+        for n, c in self.get_config()['services'].items():
+            check.state(n.startswith(self._prefix))
+            ret[n[len(self._prefix):]] = c
+
+        return ret
+
+
+def get_compose_port(cfg: ta.Mapping[str, ta.Any], default: int) -> int:
+    return check.single(
+        int(l)
+        for p in cfg['ports']
+        for l, r in [p.split(':')]
+        if int(r) == default
+    )

omlish/docker/helpers.py

@@ -0,0 +1,48 @@
+import os
+import re
+import shlex
+import sys
+
+
+##
+
+
+_DEFAULT_TIMEBOMB_NAME = '-'.join([*__name__.split('.'), 'timebomb'])
+
+
+def timebomb_payload(delay_s: float, name: str = _DEFAULT_TIMEBOMB_NAME) -> str:
+    return (
+        '('
+        f'echo {shlex.quote(name)} && '
+        f'sleep {delay_s:g} && '
+        'sh -c \'killall5 -9 -o $PPID -o $$ ; kill 1\''
+        ') &'
+    )
+
+
+##
+
+
+DOCKER_FOR_MAC_HOSTNAME = 'docker.for.mac.localhost'
+
+
+_LIKELY_IN_DOCKER_PATTERN = re.compile(r'^overlay / .*/docker/')
+
+
+def is_likely_in_docker() -> bool:
+    if getattr(sys, 'platform') != 'linux':
+        return False
+    with open('/proc/mounts') as f:
+        ls = f.readlines()
+    return any(_LIKELY_IN_DOCKER_PATTERN.match(l) for l in ls)
+
+
+##
+
+
+# Set by pyproject, docker-dev script
+DOCKER_HOST_PLATFORM_KEY = 'DOCKER_HOST_PLATFORM'
+
+
+def get_docker_host_platform() -> str | None:
+    return os.environ.get(DOCKER_HOST_PLATFORM_KEY)

omlish/docker/hub.py

@@ -0,0 +1,75 @@
+import typing as ta
+import urllib.request
+
+from .. import dataclasses as dc
+from ..formats import json
+
+
+@dc.dataclass(frozen=True)
+class HubRepoInfo:
+    repo: str
+    tags: ta.Mapping[str, ta.Any]
+    latest_manifests: ta.Mapping[str, ta.Any]
+
+
+def get_hub_repo_info(
+        repo: str,
+        *,
+        auth_url: str = 'https://auth.docker.io/',
+        api_url: str = 'https://registry-1.docker.io/v2/',
+) -> HubRepoInfo:
+    """
+    https://stackoverflow.com/a/39376254
+
+    ==
+
+    repo=library/nginx
+    token=$(
+        curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" \
+        | jq -r '.token' \
+    )
+    curl -H "Authorization: Bearer $token" -s "https://registry-1.docker.io/v2/${repo}/tags/list" | jq
+    curl \
+        -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
+        -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
+        -H "Authorization: Bearer $token" \
+        -s "https://registry-1.docker.io/v2/${repo}/manifests/latest" \
+    | jq .
+    """
+
+    auth_url = auth_url.rstrip('/')
+    api_url = api_url.rstrip('/')
+
+    #
+
+    def req_json(url: str, **kwargs: ta.Any) -> ta.Any:
+        with urllib.request.urlopen(urllib.request.Request(url, **kwargs)) as resp:  # noqa
+            return json.loads(resp.read().decode('utf-8'))
+
+    #
+
+    token_dct = req_json(f'{auth_url}/token?service=registry.docker.io&scope=repository:{repo}:pull')
+    token = token_dct['token']
+
+    req_hdrs = {'Authorization': f'Bearer {token}'}
+
+    #
+
+    tags_dct = req_json(
+        f'{api_url}/{repo}/tags/list',
+        headers=req_hdrs,
+    )
+
+    latest_mani_dct = req_json(
+        f'{api_url}/{repo}/manifests/latest',
+        headers={
+            **req_hdrs,
+            'Accept': 'application/vnd.docker.distribution.manifest.v2+json',
+        },
+    )
+
+    return HubRepoInfo(
+        repo,
+        tags_dct,
+        latest_mani_dct,
+    )

omlish/docker/manifests.py

@@ -0,0 +1,166 @@
+"""
+https://github.com/openshift/docker-distribution/blob/master/docs/spec/manifest-v2-2.md
+"""
+import typing as ta
+
+from .. import dataclasses as dc
+from .. import lang
+from .. import marshal as msh
+
+
+SCHEMA_VERSION = 2
+
+
+class MediaTypes(lang.Namespace):
+    # schema1 (existing manifest format)
+    MANIFEST_V1 = 'application/vnd.docker.distribution.manifest.v1+json'
+
+    # New image manifest format (schemaVersion = 2)
+    MANIFEST_V2 = 'application/vnd.docker.distribution.manifest.v2+json'
+
+    # Manifest list, aka "fat manifest"
+    MANIFEST_LIST = 'application/vnd.docker.distribution.manifest.list.v2+json'
+
+    # Container config JSON
+    CONTAINER_CONFIG = 'application/vnd.docker.container.image.v1+json'
+
+    # "Layer", as a gzipped tar
+    LAYER = 'application/vnd.docker.image.rootfs.diff.tar.gzip'
+
+    # "Layer", as a gzipped tar that should never be pushed
+    LAYER_NEVER_PUSH = 'application/vnd.docker.image.rootfs.foreign.diff.tar.gzip'
+
+    # Plugin config JSON
+    PLUGIN_CONFIG = 'application/vnd.docker.plugin.v1+json'
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+@msh.update_fields_metadata(['os_version'], name='os.version')
+@msh.update_fields_metadata(['os_features'], name='os.features')
+class Platform(lang.Final):
+    # The architecture field specifies the CPU architecture, for example amd64 or ppc64le.
+    architecture: str
+
+    # The os field specifies the operating system, for example linux or windows.
+    os: str
+
+    # The optional os.version field specifies the operating system version, for example 10.0.10586.
+    os_version: str | None = None
+
+    # The optional os.features field specifies an array of strings, each listing a required OS feature (for example on
+    # Windows win32k).
+    os_features: ta.Sequence[ta.Any] | None = None
+
+    # The optional variant field specifies a variant of the CPU, for example v6 to specify a particular CPU variant of
+    # the ARM CPU.
+    variant: str | None = None
+
+    # The optional features field specifies an array of strings, each listing a required CPU feature (for example sse4
+    # or aes).
+    features: ta.Sequence[ta.Any] | None = None
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+@dc.dataclass(frozen=True)
+@msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+class Manifest(lang.Final):
+    # The MIME type of the referenced object. This will generally be
+    # application/vnd.docker.distribution.manifest.v2+json, but it could also be
+    # application/vnd.docker.distribution.manifest.v1+json if the manifest list references a legacy schema-1 manifest.
+    media_type: str
+
+    # The size in bytes of the object. This field exists so that a client will have an expected size for the content
+    # before validating. If the length of the retrieved content does not match the specified length, the content should
+    # not be trusted.
+    size: int
+
+    # The digest of the content, as defined by the Registry V2 HTTP API Specification.
+    digest: str
+
+    # The platform object describes the platform which the image in the manifest runs on. A full list of valid operating
+    # system and architecture values are listed in the Go language documentation for $GOOS and $GOARCH
+    platform: Platform | None = None
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+@dc.dataclass(frozen=True)
+@msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+class ManifestList(lang.Final):
+    # This field specifies the image manifest schema version as an integer. This schema uses the version 2.
+    schema_version: int
+    dc.validate(lambda self: self.schema_version == SCHEMA_VERSION)
+
+    # The MIME type of the manifest list. This should be set to
+    # application/vnd.docker.distribution.manifest.list.v2+json.
+    media_type: str
+    dc.validate(lambda self: self.media_type == MediaTypes.MANIFEST_LIST)
+
+    # The manifests field contains a list of manifests for specific platforms.
+    manifests: ta.Sequence[Manifest]
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+@dc.dataclass(frozen=True)
+@msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+class ImageManifest(lang.Final):
+    # This field specifies the image manifest schema version as an integer. This schema uses version 2.
+    schema_version: int
+    dc.validate(lambda self: self.schema_version == SCHEMA_VERSION)
+
+    # The MIME type of the manifest. This should be set to application/vnd.docker.distribution.manifest.v2+json.
+    media_type: str
+    dc.validate(lambda self: self.media_type == MediaTypes.MANIFEST_V2)
+
+    @dc.dataclass(frozen=True)
+    @msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+    class Config(lang.Final):
+        # The MIME type of the referenced object. This should generally be
+        # application/vnd.docker.container.image.v1+json.
+        media_type: str
+
+        # The size in bytes of the object. This field exists so that a client will have an expected size for the content
+        # before validating. If the length of the retrieved content does not match the specified length, the content
+        # should not be trusted.
+        size: int
+
+        # The digest of the content, as defined by the Registry V2 HTTP API Specification.
+        digest: str
+
+        x: ta.Mapping[str, ta.Any] | None = None
+
+    # The config field references a configuration object for a container, by digest. This configuration item is a JSON
+    # blob that the runtime uses to set up the container. This new schema uses a tweaked version of this configuration
+    # o allow image content-addressability on the daemon side.
+    config: Config | None = None
+
+    @dc.dataclass(frozen=True)
+    @msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+    class Layer(lang.Final):
+        # The MIME type of the referenced object. This should generally be
+        # application/vnd.docker.image.rootfs.diff.tar.gzip. Layers of type
+        # application/vnd.docker.image.rootfs.foreign.diff.tar.gzip may be pulled from a remote location but they should
+        # never be pushed.
+        media_type: str
+
+        # The size in bytes of the object. This field exists so that a client will have an expected size for the content
+        # before validating. If the length of the retrieved content does not match the specified length, the content
+        # should not be trusted.
+        size: int
+
+        # The digest of the content, as defined by the Registry V2 HTTP API Specification.
+        digest: str
+
+        # Provides a list of URLs from which the content may be fetched. Content must be verified against the digest and
+        # size. This field is optional and uncommon.
+        urls: ta.Sequence[str] | None = None
+
+        x: ta.Mapping[str, ta.Any] | None = None
+
+    # The layer list is ordered starting from the base image (opposite order of schema1).
+    layers: ta.Sequence[Layer] | None = None
+
+    x: ta.Mapping[str, ta.Any] | None = None

omlish/lang/__init__.py

@@ -192,6 +192,7 @@ from .strings import (  # noqa
     is_sunder,
     prefix_delimited,
     prefix_lines,
+    replace_many,
     snake_case,
     strip_prefix,
     strip_suffix,