omdev 0.0.0.dev209__py3-none-any.whl → 0.0.0.dev211__py3-none-any.whl

omdev/ci/cli.py

@@ -0,0 +1,189 @@
+# @omlish-amalg ../scripts/ci.py
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+Inputs:
+ - requirements.txt
+ - ci.Dockerfile
+ - compose.yml
+
+==
+
+./python -m ci run --cache-dir ci/cache ci/project omlish-ci
+"""
+import asyncio
+import os.path
+import sys
+import typing as ta
+
+from omlish.argparse.cli import ArgparseCli
+from omlish.argparse.cli import argparse_arg
+from omlish.argparse.cli import argparse_cmd
+from omlish.lite.check import check
+from omlish.logs.standard import configure_standard_logging
+
+from .cache import DirectoryFileCache
+from .cache import DirectoryShellCache
+from .cache import FileCache
+from .cache import ShellCache
+from .ci import Ci
+from .compose import get_compose_service_dependencies
+from .github.cache import GithubShellCache
+from .github.cli import GithubCli
+from .requirements import build_requirements_hash
+from .shell import ShellCmd
+
+
+class CiCli(ArgparseCli):
+    #
+
+    @argparse_cmd(
+        argparse_arg('requirements-txt', nargs='+'),
+    )
+    def print_requirements_hash(self) -> None:
+        requirements_txts = self.args.requirements_txt
+
+        print(build_requirements_hash(requirements_txts))
+
+    #
+
+    @argparse_cmd(
+        argparse_arg('compose-file'),
+        argparse_arg('service'),
+    )
+    def dump_compose_deps(self) -> None:
+        compose_file = self.args.compose_file
+        service = self.args.service
+
+        print(get_compose_service_dependencies(
+            compose_file,
+            service,
+        ))
+
+    #
+
+    @argparse_cmd(
+        accepts_unknown=True,
+    )
+    def github(self) -> ta.Optional[int]:
+        return GithubCli(self.unknown_args).cli_run()
+
+    #
+
+    @argparse_cmd(
+        argparse_arg('project-dir'),
+        argparse_arg('service'),
+        argparse_arg('--docker-file'),
+        argparse_arg('--compose-file'),
+        argparse_arg('-r', '--requirements-txt', action='append'),
+        argparse_arg('--github-cache', action='store_true'),
+        argparse_arg('--cache-dir'),
+        argparse_arg('--always-pull', action='store_true'),
+    )
+    async def run(self) -> None:
+        project_dir = self.args.project_dir
+        docker_file = self.args.docker_file
+        compose_file = self.args.compose_file
+        service = self.args.service
+        requirements_txts = self.args.requirements_txt
+        cache_dir = self.args.cache_dir
+        always_pull = self.args.always_pull
+
+        #
+
+        check.state(os.path.isdir(project_dir))
+
+        #
+
+        def find_alt_file(*alts: str) -> ta.Optional[str]:
+            for alt in alts:
+                alt_file = os.path.abspath(os.path.join(project_dir, alt))
+                if os.path.isfile(alt_file):
+                    return alt_file
+            return None
+
+        if docker_file is None:
+            docker_file = find_alt_file(
+                'docker/ci/Dockerfile',
+                'docker/ci.Dockerfile',
+                'ci.Dockerfile',
+                'Dockerfile',
+            )
+        check.state(os.path.isfile(docker_file))
+
+        if compose_file is None:
+            compose_file = find_alt_file(
+                'docker/compose.yml',
+                'compose.yml',
+            )
+        check.state(os.path.isfile(compose_file))
+
+        if not requirements_txts:
+            requirements_txts = []
+            for rf in [
+                'requirements.txt',
+                'requirements-dev.txt',
+                'requirements-ci.txt',
+            ]:
+                if os.path.exists(os.path.join(project_dir, rf)):
+                    requirements_txts.append(rf)
+        else:
+            for rf in requirements_txts:
+                check.state(os.path.isfile(rf))
+
+        #
+
+        shell_cache: ta.Optional[ShellCache] = None
+        file_cache: ta.Optional[FileCache] = None
+        if cache_dir is not None:
+            if not os.path.exists(cache_dir):
+                os.makedirs(cache_dir)
+            check.state(os.path.isdir(cache_dir))
+
+            directory_file_cache = DirectoryFileCache(cache_dir)
+
+            file_cache = directory_file_cache
+
+            if self.args.github_cache:
+                shell_cache = GithubShellCache(cache_dir)
+            else:
+                shell_cache = DirectoryShellCache(directory_file_cache)
+
+        #
+
+        with Ci(
+                Ci.Config(
+                    project_dir=project_dir,
+
+                    docker_file=docker_file,
+
+                    compose_file=compose_file,
+                    service=service,
+
+                    requirements_txts=requirements_txts,
+
+                    cmd=ShellCmd(' && '.join([
+                        'cd /project',
+                        'python3 -m pytest -svv test.py',
+                    ])),
+
+                    always_pull=always_pull,
+                ),
+                file_cache=file_cache,
+                shell_cache=shell_cache,
+        ) as ci:
+            ci.run()
+
+
+async def _async_main() -> ta.Optional[int]:
+    return await CiCli().async_cli_run()
+
+
+def _main() -> None:
+    configure_standard_logging('DEBUG')
+
+    sys.exit(rc if isinstance(rc := asyncio.run(_async_main()), int) else 0)
+
+
+if __name__ == '__main__':
+    _main()

omdev/ci/compose.py

@@ -0,0 +1,214 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - fix rmi - only when not referenced anymore
+"""
+import contextlib
+import dataclasses as dc
+import itertools
+import os.path
+import shlex
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import ExitStacked
+from omlish.lite.contextmanagers import defer
+from omlish.lite.json import json_dumps_pretty
+from omlish.subprocesses import subprocesses
+
+from .shell import ShellCmd
+from .utils import make_temp_file
+from .utils import read_yaml_file
+
+
+##
+
+
+def get_compose_service_dependencies(
+        compose_file: str,
+        service: str,
+) -> ta.Dict[str, str]:
+    compose_dct = read_yaml_file(compose_file)
+
+    services = compose_dct['services']
+    service_dct = services[service]
+
+    out = {}
+    for dep_service in service_dct.get('depends_on', []):
+        dep_service_dct = services[dep_service]
+        out[dep_service] = dep_service_dct['image']
+
+    return out
+
+
+##
+
+
+class DockerComposeRun(ExitStacked):
+    @dc.dataclass(frozen=True)
+    class Config:
+        compose_file: str
+        service: str
+
+        image: str
+
+        cmd: ShellCmd
+
+        #
+
+        run_options: ta.Optional[ta.Sequence[str]] = None
+
+        cwd: ta.Optional[str] = None
+
+        #
+
+        no_dependency_cleanup: bool = False
+
+        #
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.run_options, str)
+
+    def __init__(self, cfg: Config) -> None:
+        super().__init__()
+
+        self._cfg = cfg
+
+        self._subprocess_kwargs = {
+            **(dict(cwd=self._cfg.cwd) if self._cfg.cwd is not None else {}),
+        }
+
+    #
+
+    @property
+    def image_tag(self) -> str:
+        pfx = 'sha256:'
+        if (image := self._cfg.image).startswith(pfx):
+            image = image[len(pfx):]
+
+        return f'{self._cfg.service}:{image}'
+
+    @cached_nullary
+    def tag_image(self) -> str:
+        image_tag = self.image_tag
+
+        subprocesses.check_call(
+            'docker',
+            'tag',
+            self._cfg.image,
+            image_tag,
+            **self._subprocess_kwargs,
+        )
+
+        def delete_tag() -> None:
+            subprocesses.check_call(
+                'docker',
+                'rmi',
+                image_tag,
+                **self._subprocess_kwargs,
+            )
+
+        self._enter_context(defer(delete_tag))  # noqa
+
+        return image_tag
+
+    #
+
+    def _rewrite_compose_dct(self, in_dct: ta.Dict[str, ta.Any]) -> ta.Dict[str, ta.Any]:
+        out = dict(in_dct)
+
+        #
+
+        in_services = in_dct['services']
+        out['services'] = out_services = {}
+
+        #
+
+        in_service: dict = in_services[self._cfg.service]
+        out_services[self._cfg.service] = out_service = dict(in_service)
+
+        out_service['image'] = self.image_tag
+
+        for k in ['build', 'platform']:
+            if k in out_service:
+                del out_service[k]
+
+        out_service['links'] = [
+            f'{l}:{l}' if ':' not in l else l
+            for l in out_service.get('links', [])
+        ]
+
+        #
+
+        depends_on = in_service.get('depends_on', [])
+
+        for dep_service, in_dep_service_dct in list(in_services.items()):
+            if dep_service not in depends_on:
+                continue
+
+            out_dep_service: dict = dict(in_dep_service_dct)
+            out_services[dep_service] = out_dep_service
+
+            out_dep_service['ports'] = []
+
+        #
+
+        return out
+
+    @cached_nullary
+    def rewrite_compose_file(self) -> str:
+        in_dct = read_yaml_file(self._cfg.compose_file)
+
+        out_dct = self._rewrite_compose_dct(in_dct)
+
+        #
+
+        out_compose_file = make_temp_file()
+        self._enter_context(defer(lambda: os.unlink(out_compose_file)))  # noqa
+
+        compose_json = json_dumps_pretty(out_dct)
+
+        with open(out_compose_file, 'w') as f:
+            f.write(compose_json)
+
+        return out_compose_file
+
+    #
+
+    def _cleanup_dependencies(self) -> None:
+        subprocesses.check_call(
+            'docker',
+            'compose',
+            '-f', self.rewrite_compose_file(),
+            'down',
+        )
+
+    def run(self) -> None:
+        self.tag_image()
+
+        compose_file = self.rewrite_compose_file()
+
+        with contextlib.ExitStack() as es:
+            if not self._cfg.no_dependency_cleanup:
+                es.enter_context(defer(self._cleanup_dependencies))  # noqa
+
+            sh_cmd = ' '.join([
+                'docker',
+                'compose',
+                '-f', compose_file,
+                'run',
+                '--rm',
+                *itertools.chain.from_iterable(['-e', k] for k in (self._cfg.cmd.env or [])),
+                *(self._cfg.run_options or []),
+                self._cfg.service,
+                'sh', '-c', shlex.quote(self._cfg.cmd.s),
+            ])
+
+            run_cmd = dc.replace(self._cfg.cmd, s=sh_cmd)
+
+            run_cmd.run(
+                subprocesses.check_call,
+                **self._subprocess_kwargs,
+            )

omdev/ci/docker.py

@@ -0,0 +1,151 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - some less stupid Dockerfile hash
+  - doesn't change too much though
+"""
+import contextlib
+import dataclasses as dc
+import json
+import os.path
+import shlex
+import tarfile
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import defer
+from omlish.subprocesses import subprocesses
+
+from .shell import ShellCmd
+from .utils import make_temp_file
+from .utils import sha256_str
+
+
+##
+
+
+def build_docker_file_hash(docker_file: str) -> str:
+    with open(docker_file) as f:
+        contents = f.read()
+
+    return sha256_str(contents)
+
+
+##
+
+
+def read_docker_tar_image_tag(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
+            m = mf.read()
+
+    manifests = json.loads(m.decode('utf-8'))
+    manifest = check.single(manifests)
+    tag = check.non_empty_str(check.single(manifest['RepoTags']))
+    return tag
+
+
+def read_docker_tar_image_id(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
+            i = mf.read()
+
+    index = json.loads(i.decode('utf-8'))
+    manifest = check.single(index['manifests'])
+    image_id = check.non_empty_str(manifest['digest'])
+    return image_id
+
+
+##
+
+
+def is_docker_image_present(image: str) -> bool:
+    out = subprocesses.check_output(
+        'docker',
+        'images',
+        '--format', 'json',
+        image,
+    )
+
+    out_s = out.decode('utf-8').strip()
+    if not out_s:
+        return False
+
+    json.loads(out_s)  # noqa
+    return True
+
+
+def pull_docker_image(
+        image: str,
+) -> None:
+    subprocesses.check_call(
+        'docker',
+        'pull',
+        image,
+    )
+
+
+def build_docker_image(
+        docker_file: str,
+        *,
+        cwd: ta.Optional[str] = None,
+) -> str:
+    id_file = make_temp_file()
+    with defer(lambda: os.unlink(id_file)):
+        subprocesses.check_call(
+            'docker',
+            'build',
+            '-f', os.path.abspath(docker_file),
+            '--iidfile', id_file,
+            '--squash',
+            '.',
+            **(dict(cwd=cwd) if cwd is not None else {}),
+        )
+
+        with open(id_file) as f:
+            image_id = check.single(f.read().strip().splitlines()).strip()
+
+    return image_id
+
+
+##
+
+
+def save_docker_tar_cmd(
+        image: str,
+        output_cmd: ShellCmd,
+) -> None:
+    cmd = dc.replace(output_cmd, s=f'docker save {image} | {output_cmd.s}')
+    cmd.run(subprocesses.check_call)
+
+
+def save_docker_tar(
+        image: str,
+        tar_file: str,
+) -> None:
+    return save_docker_tar_cmd(
+        image,
+        ShellCmd(f'cat > {shlex.quote(tar_file)}'),
+    )
+
+
+#
+
+
+def load_docker_tar_cmd(
+        input_cmd: ShellCmd,
+) -> str:
+    cmd = dc.replace(input_cmd, s=f'{input_cmd.s} | docker load')
+
+    out = cmd.run(subprocesses.check_output).decode()
+
+    line = check.single(out.strip().splitlines())
+    loaded = line.partition(':')[2].strip()
+    return loaded
+
+
+def load_docker_tar(
+        tar_file: str,
+) -> str:
+    return load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))

omdev/ci/github/bootstrap.py

@@ -0,0 +1,11 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+sudo rm -rf \
+    /usr/local/.ghcup \
+    /opt/hostedtoolcache \
+
+/usr/local/.ghcup       6.4G, 3391250 files
+/opt/hostedtoolcache    8.0G, 14843980 files
+/usr/local/lib/android  6.4G, 17251667 files
+"""