omdev 0.0.0.dev209__py3-none-any.whl → 0.0.0.dev210__py3-none-any.whl

omdev/ci/compose.py

@@ -0,0 +1,198 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - fix rmi - only when not referenced anymore
+"""
+import dataclasses as dc
+import os.path
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import ExitStacked
+from omlish.lite.contextmanagers import defer
+from omlish.lite.json import json_dumps_pretty
+from omlish.subprocesses import subprocesses
+
+from .utils import make_temp_file
+from .utils import read_yaml_file
+
+
+##
+
+
+def get_compose_service_dependencies(
+        compose_file: str,
+        service: str,
+) -> ta.Dict[str, str]:
+    compose_dct = read_yaml_file(compose_file)
+
+    services = compose_dct['services']
+    service_dct = services[service]
+
+    out = {}
+    for dep_service in service_dct.get('depends_on', []):
+        dep_service_dct = services[dep_service]
+        out[dep_service] = dep_service_dct['image']
+
+    return out
+
+
+##
+
+
+class DockerComposeRun(ExitStacked):
+    @dc.dataclass(frozen=True)
+    class Config:
+        compose_file: str
+        service: str
+
+        image: str
+
+        run_cmd: ta.Sequence[str]
+
+        #
+
+        run_options: ta.Optional[ta.Sequence[str]] = None
+
+        cwd: ta.Optional[str] = None
+
+        #
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.run_cmd, str)
+
+            check.not_isinstance(self.run_options, str)
+
+    def __init__(self, cfg: Config) -> None:
+        super().__init__()
+
+        self._cfg = cfg
+
+        self._subprocess_kwargs = {
+            **(dict(cwd=self._cfg.cwd) if self._cfg.cwd is not None else {}),
+        }
+
+    #
+
+    @property
+    def image_tag(self) -> str:
+        pfx = 'sha256:'
+        if (image := self._cfg.image).startswith(pfx):
+            image = image[len(pfx):]
+
+        return f'{self._cfg.service}:{image}'
+
+    @cached_nullary
+    def tag_image(self) -> str:
+        image_tag = self.image_tag
+
+        subprocesses.check_call(
+            'docker',
+            'tag',
+            self._cfg.image,
+            image_tag,
+            **self._subprocess_kwargs,
+        )
+
+        def delete_tag() -> None:
+            subprocesses.check_call(
+                'docker',
+                'rmi',
+                image_tag,
+                **self._subprocess_kwargs,
+            )
+
+        self._enter_context(defer(delete_tag))  # noqa
+
+        return image_tag
+
+    #
+
+    def _rewrite_compose_dct(self, in_dct: ta.Dict[str, ta.Any]) -> ta.Dict[str, ta.Any]:
+        out = dict(in_dct)
+
+        #
+
+        in_services = in_dct['services']
+        out['services'] = out_services = {}
+
+        #
+
+        in_service: dict = in_services[self._cfg.service]
+        out_services[self._cfg.service] = out_service = dict(in_service)
+
+        out_service['image'] = self.image_tag
+
+        for k in ['build', 'platform']:
+            if k in out_service:
+                del out_service[k]
+
+        out_service['links'] = [
+            f'{l}:{l}' if ':' not in l else l
+            for l in out_service.get('links', [])
+        ]
+
+        #
+
+        depends_on = in_service.get('depends_on', [])
+
+        for dep_service, in_dep_service_dct in list(in_services.items()):
+            if dep_service not in depends_on:
+                continue
+
+            out_dep_service: dict = dict(in_dep_service_dct)
+            out_services[dep_service] = out_dep_service
+
+            out_dep_service['ports'] = []
+
+        #
+
+        return out
+
+    @cached_nullary
+    def rewrite_compose_file(self) -> str:
+        in_dct = read_yaml_file(self._cfg.compose_file)
+
+        out_dct = self._rewrite_compose_dct(in_dct)
+
+        #
+
+        out_compose_file = make_temp_file()
+        self._enter_context(defer(lambda: os.unlink(out_compose_file)))  # noqa
+
+        compose_json = json_dumps_pretty(out_dct)
+
+        with open(out_compose_file, 'w') as f:
+            f.write(compose_json)
+
+        return out_compose_file
+
+    #
+
+    def run(self) -> None:
+        self.tag_image()
+
+        compose_file = self.rewrite_compose_file()
+
+        try:
+            subprocesses.check_call(
+                'docker',
+                'compose',
+                '-f', compose_file,
+                'run',
+                '--rm',
+                *self._cfg.run_options or [],
+                self._cfg.service,
+                *self._cfg.run_cmd,
+                **self._subprocess_kwargs,
+            )
+
+        finally:
+            subprocesses.check_call(
+                'docker',
+                'compose',
+                '-f', compose_file,
+                'down',
+            )

omdev/ci/dockertars.py

@@ -0,0 +1,138 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - some less stupid Dockerfile hash
+  - doesn't change too much though
+"""
+import contextlib
+import json
+import os.path
+import tarfile
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import defer
+from omlish.subprocesses import subprocesses
+
+from .utils import make_temp_file
+from .utils import sha256_str
+
+
+##
+
+
+def build_docker_file_hash(docker_file: str) -> str:
+    with open(docker_file) as f:
+        contents = f.read()
+
+    return sha256_str(contents)
+
+
+##
+
+
+def read_docker_tar_image_tag(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
+            m = mf.read()
+
+    manifests = json.loads(m.decode('utf-8'))
+    manifest = check.single(manifests)
+    tag = check.non_empty_str(check.single(manifest['RepoTags']))
+    return tag
+
+
+def read_docker_tar_image_id(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
+            i = mf.read()
+
+    index = json.loads(i.decode('utf-8'))
+    manifest = check.single(index['manifests'])
+    image_id = check.non_empty_str(manifest['digest'])
+    return image_id
+
+
+##
+
+
+def is_docker_image_present(image: str) -> bool:
+    out = subprocesses.check_output(
+        'docker',
+        'images',
+        '--format', 'json',
+        image,
+    )
+
+    out_s = out.decode('utf-8').strip()
+    if not out_s:
+        return False
+
+    json.loads(out_s)  # noqa
+    return True
+
+
+##
+
+
+def pull_docker_tar(
+        image: str,
+        tar_file: str,
+) -> None:
+    subprocesses.check_call(
+        'docker',
+        'pull',
+        image,
+    )
+
+    subprocesses.check_call(
+        'docker',
+        'save',
+        image,
+        '-o', tar_file,
+    )
+
+
+def build_docker_tar(
+    docker_file: str,
+    tar_file: str,
+    *,
+    cwd: ta.Optional[str] = None,
+) -> str:
+    id_file = make_temp_file()
+    with defer(lambda: os.unlink(id_file)):
+        subprocesses.check_call(
+            'docker',
+            'build',
+            '-f', os.path.abspath(docker_file),
+            '--iidfile', id_file,
+            '--squash',
+            '.',
+            **(dict(cwd=cwd) if cwd is not None else {}),
+        )
+
+        with open(id_file) as f:
+            image_id = check.single(f.read().strip().splitlines()).strip()
+
+        subprocesses.check_call(
+            'docker',
+            'save',
+            image_id,
+            '-o', tar_file,
+        )
+
+        return image_id
+
+
+##
+
+
+def load_docker_tar(
+        tar_file: str,
+) -> None:
+    subprocesses.check_call(
+        'docker',
+        'load',
+        '-i', tar_file,
+    )

omdev/ci/requirements.py

@@ -0,0 +1,79 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - pip compile lol
+  - but still support git+ stuff
+ - req.txt format aware hash
+  - more than just whitespace
+ - pyproject req rewriting
+ - download_requirements bootstrap off prev? not worth the dl?
+  - big deps (torch) change less, probably worth it
+ - follow embedded -r automatically like pyp
+"""
+import itertools
+import os.path
+import shutil
+import tempfile
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import defer
+from omlish.subprocesses import subprocesses
+
+from .utils import sha256_str
+
+
+##
+
+
+def build_requirements_hash(
+        requirements_txts: ta.Sequence[str],
+) -> str:
+    txt_file_contents: dict = {}
+
+    for txt_file in requirements_txts:
+        txt_file_name = os.path.basename(txt_file)
+        check.not_in(txt_file_name, txt_file_contents)
+        with open(txt_file) as f:
+            txt_contents = f.read()
+        txt_file_contents[txt_file_name] = txt_contents
+
+    #
+
+    lines = []
+    for txt_file, txt_contents in sorted(txt_file_contents.items()):
+        txt_hash = sha256_str(txt_contents)
+        lines.append(f'{txt_file}={txt_hash}')
+
+    return sha256_str('\n'.join(lines))
+
+
+##
+
+
+def download_requirements(
+        image: str,
+        requirements_dir: str,
+        requirements_txts: ta.Sequence[str],
+) -> None:
+    requirements_txt_dir = tempfile.mkdtemp()
+    with defer(lambda: shutil.rmtree(requirements_txt_dir)):
+        for rt in requirements_txts:
+            shutil.copyfile(rt, os.path.join(requirements_txt_dir, os.path.basename(rt)))
+
+        subprocesses.check_call(
+            'docker',
+            'run',
+            '-i',
+            '-v', f'{os.path.abspath(requirements_dir)}:/requirements',
+            '-v', f'{requirements_txt_dir}:/requirements_txt',
+            image,
+            'pip',
+            'download',
+            '-d', '/requirements',
+            *itertools.chain.from_iterable([
+                ['-r', f'/requirements_txt/{os.path.basename(rt)}']
+                for rt in requirements_txts
+            ]),
+        )

omdev/ci/utils.py

@@ -0,0 +1,32 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import hashlib
+import os.path
+import tempfile
+import typing as ta
+
+
+##
+
+
+def make_temp_file() -> str:
+    file_fd, file = tempfile.mkstemp()
+    os.close(file_fd)
+    return file
+
+
+##
+
+
+def read_yaml_file(yaml_file: str) -> ta.Any:
+    yaml = __import__('yaml')
+
+    with open(yaml_file) as f:
+        return yaml.safe_load(f)
+
+
+##
+
+
+def sha256_str(s: str) -> str:
+    return hashlib.sha256(s.encode('utf-8')).hexdigest()

omdev/interp/cli.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # @omlish-amalg ../scripts/interp.py
 # ruff: noqa: UP007
 """

omdev/interp/providers/system.py

@@ -104,7 +104,8 @@ class SystemInterpProvider(InterpProvider):
     async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
         if not self._options.inspect:
             s = os.path.basename(exe)
-            s = s.removeprefix('python')
+            if s.startswith('python'):  # noqa
+                s = s[len('python'):]
             if '.' in s:
                 try:
                     return InterpVersion.parse(s)

omdev/pycharm/cli.py

@@ -88,7 +88,7 @@ class Cli(ap.Cli):
 
     @ap.cmd(
         ap.arg('dir', nargs='?'),
-        ap.arg('--clion', action='store_true'),
+        ap.arg('-c', '--clion', action='store_true'),
     )
     def open(self) -> None:
         dir = os.path.abspath(self.args.dir or '.')  # noqa

omdev/pyproject/cli.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # @omlish-amalg ../scripts/pyproject.py
 # ruff: noqa: UP006 UP007
 """

omdev/revisions.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 """
 TODO:
  - omlish-lite, move to pyproject/