omdev 0.0.0.dev212__py3-none-any.whl → 0.0.0.dev213__py3-none-any.whl

omdev/ci/docker.py

@@ -13,9 +13,9 @@ import shlex
 import tarfile
 import typing as ta
 
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 from omlish.lite.contextmanagers import defer
-from omlish.subprocesses import subprocesses
 
 from .shell import ShellCmd
 from .utils import make_temp_file
@@ -60,8 +60,8 @@ def read_docker_tar_image_id(tar_file: str) -> str:
 ##
 
 
-def is_docker_image_present(image: str) -> bool:
-    out = subprocesses.check_output(
+async def is_docker_image_present(image: str) -> bool:
+    out = await asyncio_subprocesses.check_output(
         'docker',
         'images',
         '--format', 'json',
@@ -76,55 +76,74 @@ def is_docker_image_present(image: str) -> bool:
     return True
 
 
-def pull_docker_image(
+async def pull_docker_image(
         image: str,
 ) -> None:
-    subprocesses.check_call(
+    await asyncio_subprocesses.check_call(
         'docker',
         'pull',
         image,
     )
 
 
-def build_docker_image(
+async def build_docker_image(
         docker_file: str,
         *,
+        tag: ta.Optional[str] = None,
         cwd: ta.Optional[str] = None,
 ) -> str:
     id_file = make_temp_file()
     with defer(lambda: os.unlink(id_file)):
-        subprocesses.check_call(
+        await asyncio_subprocesses.check_call(
             'docker',
             'build',
             '-f', os.path.abspath(docker_file),
             '--iidfile', id_file,
             '--squash',
+            *(['--tag', tag] if tag is not None else []),
             '.',
             **(dict(cwd=cwd) if cwd is not None else {}),
         )
 
-        with open(id_file) as f:
+        with open(id_file) as f:  # noqa
             image_id = check.single(f.read().strip().splitlines()).strip()
 
     return image_id
 
 
+async def tag_docker_image(image: str, tag: str) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'tag',
+        image,
+        tag,
+    )
+
+
+async def delete_docker_tag(tag: str) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'rmi',
+        tag,
+    )
+
+
 ##
 
 
-def save_docker_tar_cmd(
+async def save_docker_tar_cmd(
         image: str,
         output_cmd: ShellCmd,
 ) -> None:
     cmd = dc.replace(output_cmd, s=f'docker save {image} | {output_cmd.s}')
-    cmd.run(subprocesses.check_call)
+    await cmd.run(asyncio_subprocesses.check_call)
 
 
-def save_docker_tar(
+async def save_docker_tar(
         image: str,
         tar_file: str,
 ) -> None:
-    return save_docker_tar_cmd(
+    return await save_docker_tar_cmd(
         image,
         ShellCmd(f'cat > {shlex.quote(tar_file)}'),
     )
@@ -133,19 +152,19 @@ def save_docker_tar(
 #
 
 
-def load_docker_tar_cmd(
+async def load_docker_tar_cmd(
         input_cmd: ShellCmd,
 ) -> str:
     cmd = dc.replace(input_cmd, s=f'{input_cmd.s} | docker load')
 
-    out = cmd.run(subprocesses.check_output).decode()
+    out = (await cmd.run(asyncio_subprocesses.check_output)).decode()
 
     line = check.single(out.strip().splitlines())
     loaded = line.partition(':')[2].strip()
     return loaded
 
 
-def load_docker_tar(
+async def load_docker_tar(
         tar_file: str,
 ) -> str:
-    return load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))
+    return await load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))

omdev/ci/github/cache.py

@@ -1,14 +1,13 @@
 # ruff: noqa: UP006 UP007
 # @omlish-lite
+import abc
 import dataclasses as dc
-import json
 import os
 import shlex
 import typing as ta
+import urllib.parse
 
 from omlish.lite.check import check
-from omlish.lite.contextmanagers import defer
-from omlish.lite.json import json_dumps_compact
 from omlish.subprocesses import subprocesses
 
 from ..cache import DirectoryFileCache
@@ -16,256 +15,168 @@ from ..cache import ShellCache
 from ..shell import ShellCmd
 from ..utils import make_temp_file
 from .cacheapi import GithubCacheServiceV1
+from .curl import GithubServiceCurlClient
 
 
 ##
 
 
-class GithubV1CacheShellClient:
+class GithubCacheShellClient(abc.ABC):
+    class Entry(abc.ABC):  # noqa
+        pass
+
+    @abc.abstractmethod
+    def run_get_entry(self, key: str) -> ta.Optional[Entry]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def download_get_entry(self, entry: Entry, out_file: str) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def upload_cache_entry(self, key: str, in_file: str) -> None:
+        raise NotImplementedError
+
+
+#
+
+
+class GithubCacheServiceV1ShellClient(GithubCacheShellClient):
     BASE_URL_ENV_KEY = 'ACTIONS_CACHE_URL'
     AUTH_TOKEN_ENV_KEY = 'ACTIONS_RUNTIME_TOKEN'  # noqa
 
+    KEY_SUFFIX_ENV_KEY = 'GITHUB_RUN_ID'
+
+    CACHE_VERSION: ta.ClassVar[int] = 1
+
+    #
+
     def __init__(
             self,
             *,
             base_url: ta.Optional[str] = None,
             auth_token: ta.Optional[str] = None,
+
+            key_prefix: ta.Optional[str] = None,
+            key_suffix: ta.Optional[str] = None,
     ) -> None:
         super().__init__()
 
+        #
+
         if base_url is None:
             base_url = os.environ[self.BASE_URL_ENV_KEY]
-        self._base_url = check.non_empty_str(base_url)
+        service_url = GithubCacheServiceV1.get_service_url(base_url)
 
         if auth_token is None:
             auth_token = os.environ.get(self.AUTH_TOKEN_ENV_KEY)
-        self._auth_token = auth_token
-
-        self._service_url = GithubCacheServiceV1.get_service_url(self._base_url)
-
-    #
-
-    _MISSING = object()
 
-    def build_headers(
-            self,
-            *,
-            auth_token: ta.Any = _MISSING,
-            content_type: ta.Optional[str] = None,
-    ) -> ta.Dict[str, str]:
-        dct = {
-            'Accept': f'application/json;api-version={GithubCacheServiceV1.API_VERSION}',
-        }
+        self._curl = GithubServiceCurlClient(
+            service_url,
+            auth_token,
+            api_version=GithubCacheServiceV1.API_VERSION,
+        )
 
-        if auth_token is self._MISSING:
-            auth_token = self._auth_token
-        if auth_token:
-            dct['Authorization'] = f'Bearer {auth_token}'
+        #
 
-        if content_type is not None:
-            dct['Content-Type'] = content_type
+        self._key_prefix = key_prefix
 
-        return dct
+        if key_suffix is None:
+            key_suffix = os.environ[self.KEY_SUFFIX_ENV_KEY]
+        self._key_suffix = check.non_empty_str(key_suffix)
 
     #
 
-    HEADER_AUTH_TOKEN_ENV_KEY = '_GITHUB_CACHE_AUTH_TOKEN'  # noqa
-
-    def build_curl_cmd(
-            self,
-            method: str,
-            url: str,
-            *,
-            json_content: bool = False,
-            content_type: ta.Optional[str] = None,
-    ) -> ShellCmd:
-        if content_type is None and json_content:
-            content_type = 'application/json'
-
-        env = {}
-
-        header_auth_token: ta.Optional[str]
-        if self._auth_token:
-            env[self.HEADER_AUTH_TOKEN_ENV_KEY] = self._auth_token
-            header_auth_token = f'${self.HEADER_AUTH_TOKEN_ENV_KEY}'
-        else:
-            header_auth_token = None
-
-        hdrs = self.build_headers(
-            auth_token=header_auth_token,
-            content_type=content_type,
-        )
-
-        url = f'{self._service_url}/{url}'
+    KEY_PART_SEPARATOR = '--'
 
-        cmd = ' '.join([
-            'curl',
-            '-s',
-            '-X', method,
-            url,
-            *[f'-H "{k}: {v}"' for k, v in hdrs.items()],
+    def fix_key(self, s: str) -> str:
+        return self.KEY_PART_SEPARATOR.join([
+            *([self._key_prefix] if self._key_prefix else []),
+            s,
+            self._key_suffix,
         ])
 
-        return ShellCmd(
-            cmd,
-            env=env,
-        )
-
-    def build_post_json_curl_cmd(
-            self,
-            url: str,
-            obj: ta.Any,
-            **kwargs: ta.Any,
-    ) -> ShellCmd:
-        curl_cmd = self.build_curl_cmd(
-            'POST',
-            url,
-            json_content=True,
-            **kwargs,
-        )
-
-        obj_json = json_dumps_compact(obj)
-
-        return dc.replace(curl_cmd, s=f'{curl_cmd.s} -d {shlex.quote(obj_json)}')
-
     #
 
-    @dc.dataclass()
-    class CurlError(RuntimeError):
-        status_code: int
-        body: ta.Optional[bytes]
-
-        def __str__(self) -> str:
-            return repr(self)
-
     @dc.dataclass(frozen=True)
-    class CurlResult:
-        status_code: int
-        body: ta.Optional[bytes]
-
-        def as_error(self) -> 'GithubV1CacheShellClient.CurlError':
-            return GithubV1CacheShellClient.CurlError(
-                status_code=self.status_code,
-                body=self.body,
-            )
-
-    def run_curl_cmd(
-            self,
-            cmd: ShellCmd,
-            *,
-            raise_: bool = False,
-    ) -> CurlResult:
-        out_file = make_temp_file()
-        with defer(lambda: os.unlink(out_file)):
-            run_cmd = dc.replace(cmd, s=f"{cmd.s} -o {out_file} -w '%{{json}}'")
-
-            out_json_bytes = run_cmd.run(subprocesses.check_output)
-
-            out_json = json.loads(out_json_bytes.decode())
-            status_code = check.isinstance(out_json['response_code'], int)
-
-            with open(out_file, 'rb') as f:
-                body = f.read()
-
-            result = self.CurlResult(
-                status_code=status_code,
-                body=body,
-            )
-
-        if raise_ and (500 <= status_code <= 600):
-            raise result.as_error()
-
-        return result
-
-    def run_json_curl_cmd(
-            self,
-            cmd: ShellCmd,
-            *,
-            success_status_codes: ta.Optional[ta.Container[int]] = None,
-    ) -> ta.Optional[ta.Any]:
-        result = self.run_curl_cmd(cmd, raise_=True)
-
-        if success_status_codes is not None:
-            is_success = result.status_code in success_status_codes
-        else:
-            is_success = 200 <= result.status_code < 300
-
-        if is_success:
-            if not (body := result.body):
-                return None
-            return json.loads(body.decode('utf-8-sig'))
-
-        elif result.status_code == 404:
-            return None
-
-        else:
-            raise result.as_error()
+    class Entry(GithubCacheShellClient.Entry):
+        artifact: GithubCacheServiceV1.ArtifactCacheEntry
 
     #
 
     def build_get_entry_curl_cmd(self, key: str) -> ShellCmd:
-        return self.build_curl_cmd(
+        fixed_key = self.fix_key(key)
+
+        qp = dict(
+            keys=fixed_key,
+            version=str(self.CACHE_VERSION),
+        )
+
+        return self._curl.build_cmd(
             'GET',
-            f'cache?keys={key}',
+            shlex.quote('?'.join([
+                'cache',
+                '&'.join([
+                    f'{k}={urllib.parse.quote_plus(v)}'
+                    for k, v in qp.items()
+                ]),
+            ])),
         )
 
-    def run_get_entry(self, key: str) -> ta.Optional[GithubCacheServiceV1.ArtifactCacheEntry]:
-        curl_cmd = self.build_get_entry_curl_cmd(key)
+    def run_get_entry(self, key: str) -> ta.Optional[Entry]:
+        fixed_key = self.fix_key(key)
+        curl_cmd = self.build_get_entry_curl_cmd(fixed_key)
 
-        obj = self.run_json_curl_cmd(
+        obj = self._curl.run_json_cmd(
             curl_cmd,
             success_status_codes=[200, 204],
         )
         if obj is None:
             return None
 
-        return GithubCacheServiceV1.dataclass_from_json(
+        return self.Entry(GithubCacheServiceV1.dataclass_from_json(
             GithubCacheServiceV1.ArtifactCacheEntry,
             obj,
-        )
+        ))
 
     #
 
-    def build_download_get_entry_cmd(
-            self,
-            entry: GithubCacheServiceV1.ArtifactCacheEntry,
-            out_file: str,
-    ) -> ShellCmd:
+    def build_download_get_entry_cmd(self, entry: Entry, out_file: str) -> ShellCmd:
         return ShellCmd(' '.join([
             'aria2c',
             '-x', '4',
             '-o', out_file,
-            check.non_empty_str(entry.archive_location),
+            check.non_empty_str(entry.artifact.archive_location),
         ]))
 
-    def download_get_entry(
-            self,
-            entry: GithubCacheServiceV1.ArtifactCacheEntry,
-            out_file: str,
-    ) -> None:
-        dl_cmd = self.build_download_get_entry_cmd(entry, out_file)
+    def download_get_entry(self, entry: GithubCacheShellClient.Entry, out_file: str) -> None:
+        dl_cmd = self.build_download_get_entry_cmd(
+            check.isinstance(entry, GithubCacheServiceV1ShellClient.Entry),
+            out_file,
+        )
         dl_cmd.run(subprocesses.check_call)
 
     #
 
-    def upload_cache_entry(
-            self,
-            key: str,
-            in_file: str,
-    ) -> None:
+    def upload_cache_entry(self, key: str, in_file: str) -> None:
+        fixed_key = self.fix_key(key)
+
         check.state(os.path.isfile(in_file))
 
         file_size = os.stat(in_file).st_size
 
+        #
+
         reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
-            key=key,
+            key=fixed_key,
             cache_size=file_size,
+            version=str(self.CACHE_VERSION),
         )
-        reserve_cmd = self.build_post_json_curl_cmd(
+        reserve_cmd = self._curl.build_post_json_cmd(
             'caches',
             GithubCacheServiceV1.dataclass_to_json(reserve_req),
         )
-        reserve_resp_obj: ta.Any = check.not_none(self.run_json_curl_cmd(
+        reserve_resp_obj: ta.Any = check.not_none(self._curl.run_json_cmd(
             reserve_cmd,
             success_status_codes=[201],
         ))
@@ -273,8 +184,66 @@ class GithubV1CacheShellClient:
             GithubCacheServiceV1.ReserveCacheResponse,
             reserve_resp_obj,
         )
+        cache_id = check.isinstance(reserve_resp.cache_id, int)
+
+        #
+
+        tmp_file = make_temp_file()
+
+        print(f'{file_size=}')
+        num_written = 0
+        chunk_size = 32 * 1024 * 1024
+        for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
+            ofs = i * chunk_size
+            sz = min(chunk_size, file_size - ofs)
+
+            patch_cmd = self._curl.build_cmd(
+                'PATCH',
+                f'caches/{cache_id}',
+                content_type='application/octet-stream',
+                headers={
+                    'Content-Range': f'bytes {ofs}-{ofs + sz - 1}/*',
+                },
+            )
 
-        raise NotImplementedError
+            #
+
+            # patch_data_cmd = dc.replace(patch_cmd, s=' | '.join([
+            #     f'dd if={in_file} bs={chunk_size} skip={i} count=1 status=none',
+            #     f'{patch_cmd.s} --data-binary -',
+            # ]))
+            # print(f'{patch_data_cmd.s=}')
+            # patch_result = self._curl.run_cmd(patch_data_cmd, raise_=True)
+
+            #
+
+            with open(in_file, 'rb') as f:
+                f.seek(ofs)
+                buf = f.read(sz)
+            with open(tmp_file, 'wb') as f:
+                f.write(buf)
+            num_written += len(buf)
+            print(f'{num_written=}')
+            patch_data_cmd = dc.replace(patch_cmd, s=f'{patch_cmd.s} --data-binary @{tmp_file}')
+            print(f'{patch_data_cmd.s=}')
+            patch_result = self._curl.run_cmd(patch_data_cmd, raise_=True)
+
+            #
+
+            check.equal(patch_result.status_code, 204)
+            ofs += sz
+
+        #
+
+        commit_req = GithubCacheServiceV1.CommitCacheRequest(
+            size=file_size,
+        )
+        commit_cmd = self._curl.build_post_json_cmd(
+            f'caches/{cache_id}',
+            GithubCacheServiceV1.dataclass_to_json(commit_req),
+        )
+        commit_result = self._curl.run_cmd(commit_cmd, raise_=True)
+        check.equal(commit_result.status_code, 204)
 
 
 ##
@@ -285,15 +254,15 @@ class GithubShellCache(ShellCache):
             self,
             dir: str,  # noqa
             *,
-            client: ta.Optional[GithubV1CacheShellClient] = None,
+            client: ta.Optional[GithubCacheShellClient] = None,
     ) -> None:
         super().__init__()
 
         self._dir = check.not_none(dir)
 
         if client is None:
-            client = GithubV1CacheShellClient()
-        self._client = client
+            client = GithubCacheServiceV1ShellClient()
+        self._client: GithubCacheShellClient = client
 
         self._local = DirectoryFileCache(self._dir)
 

omdev/ci/github/cacheapi.py

@@ -93,7 +93,7 @@ class GithubCacheServiceV1:
     @dc.dataclass(frozen=True)
     class ReserveCacheRequest:
         key: str
-        cache_size: ta.Optional[int]
+        cache_size: ta.Optional[int] = None
         version: ta.Optional[str] = None
 
     @dc.dataclass(frozen=True)

omdev/ci/github/cli.py

@@ -11,7 +11,7 @@ from omlish.argparse.cli import argparse_arg
 from omlish.argparse.cli import argparse_cmd
 from omlish.lite.json import json_dumps_pretty
 
-from .cache import GithubV1CacheShellClient
+from .cache import GithubCacheServiceV1ShellClient
 
 
 class GithubCli(ArgparseCli):
@@ -19,7 +19,7 @@ class GithubCli(ArgparseCli):
         argparse_arg('key'),
     )
     def get_cache_entry(self) -> None:
-        shell_client = GithubV1CacheShellClient()
+        shell_client = GithubCacheServiceV1ShellClient()
         entry = shell_client.run_get_entry(self.args.key)
         if entry is None:
             return