omlish 0.0.0.dev40__py3-none-any.whl → 0.0.0.dev42__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev40'
-__revision__ = '12eed3861e67278bb050cec10f1ebd2296c0bbe9-dirty'
+__version__ = '0.0.0.dev42'
+__revision__ = '244991d9a1d24de7e5b30d0e1ea7cfd341a223ab'
 
 
 #

omlish/dataclasses/__init__.py

@@ -92,6 +92,7 @@ from .utils import (  # noqa
     chain_metadata,
     deep_replace,
     field_modifier,
+    fields_dict,
     maybe_post_init,
     opt_repr,
     update_class_metadata,

omlish/dataclasses/impl/__init__.py

@@ -21,4 +21,5 @@ TODO:
  - proto/jsonschema gen
  - enums
  - nodal
+ - embedding? forward kwargs in general? or only for replace?
 """

omlish/dataclasses/utils.py

@@ -36,6 +36,10 @@ def opt_repr(o: ta.Any) -> str | None:
 #
 
 
+def fields_dict(cls_or_instance: ta.Any) -> dict[str, dc.Field]:
+    return {f.name: f for f in dc.fields(cls_or_instance)}
+
+
 class field_modifier:  # noqa
     def __init__(self, fn: ta.Callable[[dc.Field], dc.Field]) -> None:
         super().__init__()

omlish/docker/__init__.py

@@ -0,0 +1,27 @@
+from .cli import (  # noqa
+    Inspect,
+    Port,
+    PsItem,
+    cli_inspect,
+    cli_ps,
+    has_cli,
+    parse_port,
+)
+
+from .compose import (  # noqa
+    ComposeConfig,
+    get_compose_port,
+)
+
+from .helpers import (  # noqa
+    DOCKER_FOR_MAC_HOSTNAME,
+    DOCKER_HOST_PLATFORM_KEY,
+    get_docker_host_platform,
+    is_likely_in_docker,
+    timebomb_payload,
+)
+
+from .hub import (  # noqa
+    HubRepoInfo,
+    get_hub_repo_info,
+)

omlish/docker/cli.py

@@ -0,0 +1,101 @@
+"""
+TODO:
+ - https://github.com/mag37/dockcheck/blob/3d122f2b868eb53a25a3014f0f6bd499390a3a29/dockcheck.sh
+  - https://github.com/regclient/regclient
+  - https://stackoverflow.com/questions/71409458/how-to-download-docker-image-using-http-api-using-docker-hub-credentials
+  - https://stackoverflow.com/questions/55386202/how-can-i-use-the-docker-registry-api-to-pull-information-about-a-container-get
+  - https://ops.tips/blog/inspecting-docker-image-without-pull/
+"""  # noqa
+import datetime
+import re
+import subprocess
+import typing as ta
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+from .. import marshal as msh
+from ..formats import json
+
+
+@dc.dataclass(frozen=True)
+@msh.update_object_metadata(field_naming=msh.Naming.CAMEL, unknown_field='x')
+@msh.update_fields_metadata(['id'], name='ID')
+class PsItem(lang.Final):
+    command: str
+    created_at: datetime.datetime
+    id: str
+    image: str
+    labels: str
+    local_volumes: str
+    mounts: str
+    names: str
+    networks: str
+    ports: str
+    running_for: str
+    size: str
+    state: str
+    status: str
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+class Port(ta.NamedTuple):
+    ip: str
+    from_port: int
+    to_port: int
+    proto: str
+
+
+_PORT_PAT = re.compile(r'(?P<ip>[^:]+):(?P<from_port>\d+)->(?P<to_port>\d+)/(?P<proto>\w+)')
+
+
+def parse_port(s: str) -> Port:
+    # '0.0.0.0:35221->22/tcp, 0.0.0.0:35220->8000/tcp'
+    m = check.not_none(_PORT_PAT.fullmatch(s))
+    return Port(
+        m.group('ip'),
+        int(m.group('from_port')),
+        int(m.group('to_port')),
+        m.group('proto'),
+    )
+
+
+def cli_ps() -> list[PsItem]:
+    o = subprocess.check_output([
+        'docker',
+        'ps',
+        '--no-trunc',
+        '--format', '{{json .}}',
+    ])
+
+    ret: list[PsItem] = []
+    for l in o.decode().splitlines():
+        d = json.loads(l)
+        pi = msh.unmarshal(d, PsItem)
+        ret.append(pi)
+
+    return ret
+
+
+@dc.dataclass(frozen=True)
+@msh.update_object_metadata(field_naming=msh.Naming.CAMEL, unknown_field='x')
+class Inspect(lang.Final):
+    id: str
+    created: datetime.datetime
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+def cli_inspect(ids: list[str]) -> list[Inspect]:
+    o = subprocess.check_output(['docker', 'inspect', *ids])
+    return msh.unmarshal(json.loads(o.decode()), list[Inspect])
+
+
+def has_cli() -> bool:
+    try:
+        proc = subprocess.run(['docker', '--version'])  # noqa
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        return False
+    else:
+        return not proc.returncode

omlish/docker/compose.py

@@ -0,0 +1,51 @@
+"""
+TODO:
+ - merged compose configs: https://github.com/wrmsr/bane/blob/27647abdcfb323b73e6982a5c318c7029496b203/core/dev/docker/compose.go#L38
+"""  # noqa
+import typing as ta
+
+from .. import check
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    import yaml
+else:
+    yaml = lang.proxy_import('yaml')
+
+
+class ComposeConfig:
+    def __init__(
+            self,
+            prefix: str,
+            *,
+            file_path: str | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._prefix = prefix
+        self._file_path = file_path
+
+    @lang.cached_function
+    def get_config(self) -> ta.Mapping[str, ta.Any]:
+        with open(check.not_none(self._file_path)) as f:
+            buf = f.read()
+        return yaml.safe_load(buf)
+
+    @lang.cached_function
+    def get_services(self) -> ta.Mapping[str, ta.Any]:
+        ret = {}
+        for n, c in self.get_config()['services'].items():
+            check.state(n.startswith(self._prefix))
+            ret[n[len(self._prefix):]] = c
+
+        return ret
+
+
+def get_compose_port(cfg: ta.Mapping[str, ta.Any], default: int) -> int:
+    return check.single(
+        int(l)
+        for p in cfg['ports']
+        for l, r in [p.split(':')]
+        if int(r) == default
+    )

omlish/docker/helpers.py

@@ -0,0 +1,48 @@
+import os
+import re
+import shlex
+import sys
+
+
+##
+
+
+_DEFAULT_TIMEBOMB_NAME = '-'.join([*__name__.split('.'), 'timebomb'])
+
+
+def timebomb_payload(delay_s: float, name: str = _DEFAULT_TIMEBOMB_NAME) -> str:
+    return (
+        '('
+        f'echo {shlex.quote(name)} && '
+        f'sleep {delay_s:g} && '
+        'sh -c \'killall5 -9 -o $PPID -o $$ ; kill 1\''
+        ') &'
+    )
+
+
+##
+
+
+DOCKER_FOR_MAC_HOSTNAME = 'docker.for.mac.localhost'
+
+
+_LIKELY_IN_DOCKER_PATTERN = re.compile(r'^overlay / .*/docker/')
+
+
+def is_likely_in_docker() -> bool:
+    if getattr(sys, 'platform') != 'linux':
+        return False
+    with open('/proc/mounts') as f:
+        ls = f.readlines()
+    return any(_LIKELY_IN_DOCKER_PATTERN.match(l) for l in ls)
+
+
+##
+
+
+# Set by pyproject, docker-dev script
+DOCKER_HOST_PLATFORM_KEY = 'DOCKER_HOST_PLATFORM'
+
+
+def get_docker_host_platform() -> str | None:
+    return os.environ.get(DOCKER_HOST_PLATFORM_KEY)

omlish/docker/hub.py

@@ -0,0 +1,75 @@
+import typing as ta
+import urllib.request
+
+from .. import dataclasses as dc
+from ..formats import json
+
+
+@dc.dataclass(frozen=True)
+class HubRepoInfo:
+    repo: str
+    tags: ta.Mapping[str, ta.Any]
+    latest_manifests: ta.Mapping[str, ta.Any]
+
+
+def get_hub_repo_info(
+        repo: str,
+        *,
+        auth_url: str = 'https://auth.docker.io/',
+        api_url: str = 'https://registry-1.docker.io/v2/',
+) -> HubRepoInfo:
+    """
+    https://stackoverflow.com/a/39376254
+
+    ==
+
+    repo=library/nginx
+    token=$(
+        curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" \
+        | jq -r '.token' \
+    )
+    curl -H "Authorization: Bearer $token" -s "https://registry-1.docker.io/v2/${repo}/tags/list" | jq
+    curl \
+        -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
+        -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
+        -H "Authorization: Bearer $token" \
+        -s "https://registry-1.docker.io/v2/${repo}/manifests/latest" \
+    | jq .
+    """
+
+    auth_url = auth_url.rstrip('/')
+    api_url = api_url.rstrip('/')
+
+    #
+
+    def req_json(url: str, **kwargs: ta.Any) -> ta.Any:
+        with urllib.request.urlopen(urllib.request.Request(url, **kwargs)) as resp:  # noqa
+            return json.loads(resp.read().decode('utf-8'))
+
+    #
+
+    token_dct = req_json(f'{auth_url}/token?service=registry.docker.io&scope=repository:{repo}:pull')
+    token = token_dct['token']
+
+    req_hdrs = {'Authorization': f'Bearer {token}'}
+
+    #
+
+    tags_dct = req_json(
+        f'{api_url}/{repo}/tags/list',
+        headers=req_hdrs,
+    )
+
+    latest_mani_dct = req_json(
+        f'{api_url}/{repo}/manifests/latest',
+        headers={
+            **req_hdrs,
+            'Accept': 'application/vnd.docker.distribution.manifest.v2+json',
+        },
+    )
+
+    return HubRepoInfo(
+        repo,
+        tags_dct,
+        latest_mani_dct,
+    )

omlish/docker/manifests.py

@@ -0,0 +1,166 @@
+"""
+https://github.com/openshift/docker-distribution/blob/master/docs/spec/manifest-v2-2.md
+"""
+import typing as ta
+
+from .. import dataclasses as dc
+from .. import lang
+from .. import marshal as msh
+
+
+SCHEMA_VERSION = 2
+
+
+class MediaTypes(lang.Namespace):
+    # schema1 (existing manifest format)
+    MANIFEST_V1 = 'application/vnd.docker.distribution.manifest.v1+json'
+
+    # New image manifest format (schemaVersion = 2)
+    MANIFEST_V2 = 'application/vnd.docker.distribution.manifest.v2+json'
+
+    # Manifest list, aka "fat manifest"
+    MANIFEST_LIST = 'application/vnd.docker.distribution.manifest.list.v2+json'
+
+    # Container config JSON
+    CONTAINER_CONFIG = 'application/vnd.docker.container.image.v1+json'
+
+    # "Layer", as a gzipped tar
+    LAYER = 'application/vnd.docker.image.rootfs.diff.tar.gzip'
+
+    # "Layer", as a gzipped tar that should never be pushed
+    LAYER_NEVER_PUSH = 'application/vnd.docker.image.rootfs.foreign.diff.tar.gzip'
+
+    # Plugin config JSON
+    PLUGIN_CONFIG = 'application/vnd.docker.plugin.v1+json'
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+@msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+@msh.update_fields_metadata(['os_version'], name='os.version')
+@msh.update_fields_metadata(['os_features'], name='os.features')
+class Platform(lang.Final):
+    # The architecture field specifies the CPU architecture, for example amd64 or ppc64le.
+    architecture: str
+
+    # The os field specifies the operating system, for example linux or windows.
+    os: str
+
+    # The optional os.version field specifies the operating system version, for example 10.0.10586.
+    os_version: str | None = None
+
+    # The optional os.features field specifies an array of strings, each listing a required OS feature (for example on
+    # Windows win32k).
+    os_features: ta.Sequence[ta.Any] | None = None
+
+    # The optional variant field specifies a variant of the CPU, for example v6 to specify a particular CPU variant of
+    # the ARM CPU.
+    variant: str | None = None
+
+    # The optional features field specifies an array of strings, each listing a required CPU feature (for example sse4
+    # or aes).
+    features: ta.Sequence[ta.Any] | None = None
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+@dc.dataclass(frozen=True)
+@msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+class Manifest(lang.Final):
+    # The MIME type of the referenced object. This will generally be
+    # application/vnd.docker.distribution.manifest.v2+json, but it could also be
+    # application/vnd.docker.distribution.manifest.v1+json if the manifest list references a legacy schema-1 manifest.
+    media_type: str
+
+    # The size in bytes of the object. This field exists so that a client will have an expected size for the content
+    # before validating. If the length of the retrieved content does not match the specified length, the content should
+    # not be trusted.
+    size: int
+
+    # The digest of the content, as defined by the Registry V2 HTTP API Specification.
+    digest: str
+
+    # The platform object describes the platform which the image in the manifest runs on. A full list of valid operating
+    # system and architecture values are listed in the Go language documentation for $GOOS and $GOARCH
+    platform: Platform | None = None
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+@dc.dataclass(frozen=True)
+@msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+class ManifestList(lang.Final):
+    # This field specifies the image manifest schema version as an integer. This schema uses the version 2.
+    schema_version: int
+    dc.validate(lambda self: self.schema_version == SCHEMA_VERSION)
+
+    # The MIME type of the manifest list. This should be set to
+    # application/vnd.docker.distribution.manifest.list.v2+json.
+    media_type: str
+    dc.validate(lambda self: self.media_type == MediaTypes.MANIFEST_LIST)
+
+    # The manifests field contains a list of manifests for specific platforms.
+    manifests: ta.Sequence[Manifest]
+
+    x: ta.Mapping[str, ta.Any] | None = None
+
+
+@dc.dataclass(frozen=True)
+@msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+class ImageManifest(lang.Final):
+    # This field specifies the image manifest schema version as an integer. This schema uses version 2.
+    schema_version: int
+    dc.validate(lambda self: self.schema_version == SCHEMA_VERSION)
+
+    # The MIME type of the manifest. This should be set to application/vnd.docker.distribution.manifest.v2+json.
+    media_type: str
+    dc.validate(lambda self: self.media_type == MediaTypes.MANIFEST_V2)
+
+    @dc.dataclass(frozen=True)
+    @msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+    class Config(lang.Final):
+        # The MIME type of the referenced object. This should generally be
+        # application/vnd.docker.container.image.v1+json.
+        media_type: str
+
+        # The size in bytes of the object. This field exists so that a client will have an expected size for the content
+        # before validating. If the length of the retrieved content does not match the specified length, the content
+        # should not be trusted.
+        size: int
+
+        # The digest of the content, as defined by the Registry V2 HTTP API Specification.
+        digest: str
+
+        x: ta.Mapping[str, ta.Any] | None = None
+
+    # The config field references a configuration object for a container, by digest. This configuration item is a JSON
+    # blob that the runtime uses to set up the container. This new schema uses a tweaked version of this configuration
+    # o allow image content-addressability on the daemon side.
+    config: Config | None = None
+
+    @dc.dataclass(frozen=True)
+    @msh.update_object_metadata(field_naming=msh.Naming.LOW_CAMEL, unknown_field='x')
+    class Layer(lang.Final):
+        # The MIME type of the referenced object. This should generally be
+        # application/vnd.docker.image.rootfs.diff.tar.gzip. Layers of type
+        # application/vnd.docker.image.rootfs.foreign.diff.tar.gzip may be pulled from a remote location but they should
+        # never be pushed.
+        media_type: str
+
+        # The size in bytes of the object. This field exists so that a client will have an expected size for the content
+        # before validating. If the length of the retrieved content does not match the specified length, the content
+        # should not be trusted.
+        size: int
+
+        # The digest of the content, as defined by the Registry V2 HTTP API Specification.
+        digest: str
+
+        # Provides a list of URLs from which the content may be fetched. Content must be verified against the digest and
+        # size. This field is optional and uncommon.
+        urls: ta.Sequence[str] | None = None
+
+        x: ta.Mapping[str, ta.Any] | None = None
+
+    # The layer list is ordered starting from the base image (opposite order of schema1).
+    layers: ta.Sequence[Layer] | None = None
+
+    x: ta.Mapping[str, ta.Any] | None = None

omlish/lang/__init__.py

@@ -192,6 +192,7 @@ from .strings import (  # noqa
     is_sunder,
     prefix_delimited,
     prefix_lines,
+    replace_many,
     snake_case,
     strip_prefix,
     strip_suffix,

omlish/lang/strings.py

@@ -2,63 +2,51 @@ import typing as ta
 import unicodedata
 
 
-##
-
-@ta.overload
-def prefix_delimited(s: str, p: str, d: str) -> str:
-    ...
+StrOrBytes: ta.TypeAlias = str | bytes
+StrOrBytesT = ta.TypeVar('StrOrBytesT', bound=StrOrBytes)
 
 
-@ta.overload
-def prefix_delimited(s: bytes, p: bytes, d: bytes) -> bytes:
-    ...
+##
 
 
-def prefix_delimited(s, p, d):
-    return d.join([p + l for l in s.split(d)])
+def prefix_delimited(s: StrOrBytesT, p: StrOrBytesT, d: StrOrBytesT) -> StrOrBytesT:
+    return d.join([p + l for l in s.split(d)])  # type: ignore
 
 
-def prefix_lines(s: str, p: str) -> str:
-    return prefix_delimited(s, p, '\n')
+def prefix_lines(s: StrOrBytesT, p: StrOrBytesT) -> StrOrBytesT:
+    return prefix_delimited(s, p, '\n' if isinstance(s, str) else b'\n')
 
 
-def indent_lines(s: str, num: int) -> str:
-    return prefix_lines(s, ' ' * num)
+def indent_lines(s: StrOrBytesT, num: StrOrBytesT) -> StrOrBytesT:
+    return prefix_lines(s, (' ' if isinstance(s, str) else b' ') * num)  # type: ignore
 
 
 ##
 
 
-@ta.overload
-def strip_prefix(s: str, pfx: str) -> str:
-    ...
-
-
-@ta.overload
-def strip_prefix(s: bytes, pfx: bytes) -> bytes:
-    ...
-
-
-def strip_prefix(s, pfx):
-    if not s.startswith(pfx):
+def strip_prefix(s: StrOrBytesT, pfx: StrOrBytesT) -> StrOrBytesT:
+    if not s.startswith(pfx):  # type: ignore
         raise ValueError(f'{s!r} does not start with {pfx!r}')
-    return s[len(pfx):]
+    return s[len(pfx):]  # type: ignore
 
 
-@ta.overload
-def strip_suffix(s: str, sfx: str) -> str:
-    ...
+def strip_suffix(s: StrOrBytesT, sfx: StrOrBytesT) -> StrOrBytesT:
+    if not s.endswith(sfx):  # type: ignore
+        raise ValueError(f'{s!r} does not end with {sfx!r}')
+    return s[:-len(sfx)]  # type: ignore
 
 
-@ta.overload
-def strip_suffix(s: bytes, sfx: bytes) -> bytes:
-    ...
+##
 
 
-def strip_suffix(s, sfx):
-    if not s.endswith(sfx):
-        raise ValueError(f'{s!r} does not end with {sfx!r}')
-    return s[:-len(sfx)]
+def replace_many(
+        s: StrOrBytesT,
+        old: ta.Iterable[StrOrBytesT],
+        new: StrOrBytesT, count_each: int = -1,
+) -> StrOrBytesT:
+    for o in old:
+        s = s.replace(o, new, count_each)  # type: ignore
+    return s
 
 
 ##

omlish/marshal/__init__.py

@@ -57,6 +57,7 @@ from .helpers import (  # noqa
 from .objects import (  # noqa
     FieldInfo,
     FieldMetadata,
+    FieldOptions,
     ObjectMarshaler,
     ObjectMetadata,
     ObjectUnmarshaler,