omdev 0.0.0.dev210__py3-none-any.whl → 0.0.0.dev211__py3-none-any.whl

omdev/ci/{dockertars.py → docker.py}

@@ -6,8 +6,10 @@ TODO:
   - doesn't change too much though
 """
 import contextlib
+import dataclasses as dc
 import json
 import os.path
+import shlex
 import tarfile
 import typing as ta
 
@@ -15,6 +17,7 @@ from omlish.lite.check import check
 from omlish.lite.contextmanagers import defer
 from omlish.subprocesses import subprocesses
 
+from .shell import ShellCmd
 from .utils import make_temp_file
 from .utils import sha256_str
 
@@ -73,12 +76,8 @@ def is_docker_image_present(image: str) -> bool:
     return True
 
 
-##
-
-
-def pull_docker_tar(
+def pull_docker_image(
         image: str,
-        tar_file: str,
 ) -> None:
     subprocesses.check_call(
         'docker',
@@ -86,19 +85,11 @@ def pull_docker_tar(
         image,
     )
 
-    subprocesses.check_call(
-        'docker',
-        'save',
-        image,
-        '-o', tar_file,
-    )
-
 
-def build_docker_tar(
-    docker_file: str,
-    tar_file: str,
-    *,
-    cwd: ta.Optional[str] = None,
+def build_docker_image(
+        docker_file: str,
+        *,
+        cwd: ta.Optional[str] = None,
 ) -> str:
     id_file = make_temp_file()
     with defer(lambda: os.unlink(id_file)):
@@ -115,24 +106,46 @@ def build_docker_tar(
         with open(id_file) as f:
             image_id = check.single(f.read().strip().splitlines()).strip()
 
-        subprocesses.check_call(
-            'docker',
-            'save',
-            image_id,
-            '-o', tar_file,
-        )
-
-        return image_id
+    return image_id
 
 
 ##
 
 
-def load_docker_tar(
+def save_docker_tar_cmd(
+        image: str,
+        output_cmd: ShellCmd,
+) -> None:
+    cmd = dc.replace(output_cmd, s=f'docker save {image} | {output_cmd.s}')
+    cmd.run(subprocesses.check_call)
+
+
+def save_docker_tar(
+        image: str,
         tar_file: str,
 ) -> None:
-    subprocesses.check_call(
-        'docker',
-        'load',
-        '-i', tar_file,
+    return save_docker_tar_cmd(
+        image,
+        ShellCmd(f'cat > {shlex.quote(tar_file)}'),
     )
+
+
+#
+
+
+def load_docker_tar_cmd(
+        input_cmd: ShellCmd,
+) -> str:
+    cmd = dc.replace(input_cmd, s=f'{input_cmd.s} | docker load')
+
+    out = cmd.run(subprocesses.check_output).decode()
+
+    line = check.single(out.strip().splitlines())
+    loaded = line.partition(':')[2].strip()
+    return loaded
+
+
+def load_docker_tar(
+        tar_file: str,
+) -> str:
+    return load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))

omdev/ci/github/bootstrap.py

@@ -0,0 +1,11 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+sudo rm -rf \
+    /usr/local/.ghcup \
+    /opt/hostedtoolcache \
+
+/usr/local/.ghcup       6.4G, 3391250 files
+/opt/hostedtoolcache    8.0G, 14843980 files
+/usr/local/lib/android  6.4G, 17251667 files
+"""

omdev/ci/github/cache.py

@@ -0,0 +1,355 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import json
+import os
+import shlex
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import defer
+from omlish.lite.json import json_dumps_compact
+from omlish.subprocesses import subprocesses
+
+from ..cache import DirectoryFileCache
+from ..cache import ShellCache
+from ..shell import ShellCmd
+from ..utils import make_temp_file
+from .cacheapi import GithubCacheServiceV1
+
+
+##
+
+
+class GithubV1CacheShellClient:
+    BASE_URL_ENV_KEY = 'ACTIONS_CACHE_URL'
+    AUTH_TOKEN_ENV_KEY = 'ACTIONS_RUNTIME_TOKEN'  # noqa
+
+    def __init__(
+            self,
+            *,
+            base_url: ta.Optional[str] = None,
+            auth_token: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+
+        if base_url is None:
+            base_url = os.environ[self.BASE_URL_ENV_KEY]
+        self._base_url = check.non_empty_str(base_url)
+
+        if auth_token is None:
+            auth_token = os.environ.get(self.AUTH_TOKEN_ENV_KEY)
+        self._auth_token = auth_token
+
+        self._service_url = GithubCacheServiceV1.get_service_url(self._base_url)
+
+    #
+
+    _MISSING = object()
+
+    def build_headers(
+            self,
+            *,
+            auth_token: ta.Any = _MISSING,
+            content_type: ta.Optional[str] = None,
+    ) -> ta.Dict[str, str]:
+        dct = {
+            'Accept': f'application/json;api-version={GithubCacheServiceV1.API_VERSION}',
+        }
+
+        if auth_token is self._MISSING:
+            auth_token = self._auth_token
+        if auth_token:
+            dct['Authorization'] = f'Bearer {auth_token}'
+
+        if content_type is not None:
+            dct['Content-Type'] = content_type
+
+        return dct
+
+    #
+
+    HEADER_AUTH_TOKEN_ENV_KEY = '_GITHUB_CACHE_AUTH_TOKEN'  # noqa
+
+    def build_curl_cmd(
+            self,
+            method: str,
+            url: str,
+            *,
+            json_content: bool = False,
+            content_type: ta.Optional[str] = None,
+    ) -> ShellCmd:
+        if content_type is None and json_content:
+            content_type = 'application/json'
+
+        env = {}
+
+        header_auth_token: ta.Optional[str]
+        if self._auth_token:
+            env[self.HEADER_AUTH_TOKEN_ENV_KEY] = self._auth_token
+            header_auth_token = f'${self.HEADER_AUTH_TOKEN_ENV_KEY}'
+        else:
+            header_auth_token = None
+
+        hdrs = self.build_headers(
+            auth_token=header_auth_token,
+            content_type=content_type,
+        )
+
+        url = f'{self._service_url}/{url}'
+
+        cmd = ' '.join([
+            'curl',
+            '-s',
+            '-X', method,
+            url,
+            *[f'-H "{k}: {v}"' for k, v in hdrs.items()],
+        ])
+
+        return ShellCmd(
+            cmd,
+            env=env,
+        )
+
+    def build_post_json_curl_cmd(
+            self,
+            url: str,
+            obj: ta.Any,
+            **kwargs: ta.Any,
+    ) -> ShellCmd:
+        curl_cmd = self.build_curl_cmd(
+            'POST',
+            url,
+            json_content=True,
+            **kwargs,
+        )
+
+        obj_json = json_dumps_compact(obj)
+
+        return dc.replace(curl_cmd, s=f'{curl_cmd.s} -d {shlex.quote(obj_json)}')
+
+    #
+
+    @dc.dataclass()
+    class CurlError(RuntimeError):
+        status_code: int
+        body: ta.Optional[bytes]
+
+        def __str__(self) -> str:
+            return repr(self)
+
+    @dc.dataclass(frozen=True)
+    class CurlResult:
+        status_code: int
+        body: ta.Optional[bytes]
+
+        def as_error(self) -> 'GithubV1CacheShellClient.CurlError':
+            return GithubV1CacheShellClient.CurlError(
+                status_code=self.status_code,
+                body=self.body,
+            )
+
+    def run_curl_cmd(
+            self,
+            cmd: ShellCmd,
+            *,
+            raise_: bool = False,
+    ) -> CurlResult:
+        out_file = make_temp_file()
+        with defer(lambda: os.unlink(out_file)):
+            run_cmd = dc.replace(cmd, s=f"{cmd.s} -o {out_file} -w '%{{json}}'")
+
+            out_json_bytes = run_cmd.run(subprocesses.check_output)
+
+            out_json = json.loads(out_json_bytes.decode())
+            status_code = check.isinstance(out_json['response_code'], int)
+
+            with open(out_file, 'rb') as f:
+                body = f.read()
+
+            result = self.CurlResult(
+                status_code=status_code,
+                body=body,
+            )
+
+        if raise_ and (500 <= status_code <= 600):
+            raise result.as_error()
+
+        return result
+
+    def run_json_curl_cmd(
+            self,
+            cmd: ShellCmd,
+            *,
+            success_status_codes: ta.Optional[ta.Container[int]] = None,
+    ) -> ta.Optional[ta.Any]:
+        result = self.run_curl_cmd(cmd, raise_=True)
+
+        if success_status_codes is not None:
+            is_success = result.status_code in success_status_codes
+        else:
+            is_success = 200 <= result.status_code < 300
+
+        if is_success:
+            if not (body := result.body):
+                return None
+            return json.loads(body.decode('utf-8-sig'))
+
+        elif result.status_code == 404:
+            return None
+
+        else:
+            raise result.as_error()
+
+    #
+
+    def build_get_entry_curl_cmd(self, key: str) -> ShellCmd:
+        return self.build_curl_cmd(
+            'GET',
+            f'cache?keys={key}',
+        )
+
+    def run_get_entry(self, key: str) -> ta.Optional[GithubCacheServiceV1.ArtifactCacheEntry]:
+        curl_cmd = self.build_get_entry_curl_cmd(key)
+
+        obj = self.run_json_curl_cmd(
+            curl_cmd,
+            success_status_codes=[200, 204],
+        )
+        if obj is None:
+            return None
+
+        return GithubCacheServiceV1.dataclass_from_json(
+            GithubCacheServiceV1.ArtifactCacheEntry,
+            obj,
+        )
+
+    #
+
+    def build_download_get_entry_cmd(
+            self,
+            entry: GithubCacheServiceV1.ArtifactCacheEntry,
+            out_file: str,
+    ) -> ShellCmd:
+        return ShellCmd(' '.join([
+            'aria2c',
+            '-x', '4',
+            '-o', out_file,
+            check.non_empty_str(entry.archive_location),
+        ]))
+
+    def download_get_entry(
+            self,
+            entry: GithubCacheServiceV1.ArtifactCacheEntry,
+            out_file: str,
+    ) -> None:
+        dl_cmd = self.build_download_get_entry_cmd(entry, out_file)
+        dl_cmd.run(subprocesses.check_call)
+
+    #
+
+    def upload_cache_entry(
+            self,
+            key: str,
+            in_file: str,
+    ) -> None:
+        check.state(os.path.isfile(in_file))
+
+        file_size = os.stat(in_file).st_size
+
+        reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
+            key=key,
+            cache_size=file_size,
+        )
+        reserve_cmd = self.build_post_json_curl_cmd(
+            'caches',
+            GithubCacheServiceV1.dataclass_to_json(reserve_req),
+        )
+        reserve_resp_obj: ta.Any = check.not_none(self.run_json_curl_cmd(
+            reserve_cmd,
+            success_status_codes=[201],
+        ))
+        reserve_resp = GithubCacheServiceV1.dataclass_from_json(  # noqa
+            GithubCacheServiceV1.ReserveCacheResponse,
+            reserve_resp_obj,
+        )
+
+        raise NotImplementedError
+
+
+##
+
+
+class GithubShellCache(ShellCache):
+    def __init__(
+            self,
+            dir: str,  # noqa
+            *,
+            client: ta.Optional[GithubV1CacheShellClient] = None,
+    ) -> None:
+        super().__init__()
+
+        self._dir = check.not_none(dir)
+
+        if client is None:
+            client = GithubV1CacheShellClient()
+        self._client = client
+
+        self._local = DirectoryFileCache(self._dir)
+
+    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+        local_file = self._local.get_cache_file_path(key)
+        if os.path.exists(local_file):
+            return ShellCmd(f'cat {shlex.quote(local_file)}')
+
+        if (entry := self._client.run_get_entry(key)) is None:
+            return None
+
+        tmp_file = self._local.format_incomplete_file(local_file)
+        try:
+            self._client.download_get_entry(entry, tmp_file)
+
+            os.replace(tmp_file, local_file)
+
+        except BaseException:  # noqa
+            os.unlink(tmp_file)
+
+            raise
+
+        return ShellCmd(f'cat {shlex.quote(local_file)}')
+
+    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
+        def __init__(
+                self,
+                owner: 'GithubShellCache',
+                key: str,
+                tmp_file: str,
+                local_file: str,
+        ) -> None:
+            super().__init__()
+
+            self._owner = owner
+            self._key = key
+            self._tmp_file = tmp_file
+            self._local_file = local_file
+
+        @property
+        def cmd(self) -> ShellCmd:
+            return ShellCmd(f'cat > {shlex.quote(self._tmp_file)}')
+
+        def _commit(self) -> None:
+            os.replace(self._tmp_file, self._local_file)
+
+            self._owner._client.upload_cache_entry(self._key, self._local_file)  # noqa
+
+        def _abort(self) -> None:
+            os.unlink(self._tmp_file)
+
+    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
+        local_file = self._local.get_cache_file_path(key, make_dirs=True)
+        return self._PutFileCmdContext(
+            self,
+            key,
+            self._local.format_incomplete_file(local_file),
+            local_file,
+        )

omdev/ci/github/cacheapi.py

@@ -0,0 +1,207 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+export FILE_SIZE=$(stat --format="%s" $FILE)
+
+export CACHE_ID=$(curl -s \
+  -X POST \
+  "${ACTIONS_CACHE_URL}_apis/artifactcache/caches" \
+  -H 'Content-Type: application/json' \
+  -H 'Accept: application/json;api-version=6.0-preview.1' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  -d '{"key": "'"$CACHE_KEY"'", "cacheSize": '"$FILE_SIZE"'}' \
+  | jq .cacheId)
+
+curl -s \
+  -X PATCH \
+  "${ACTIONS_CACHE_URL}_apis/artifactcache/caches/$CACHE_ID" \
+  -H 'Content-Type: application/octet-stream' \
+  -H 'Accept: application/json;api-version=6.0-preview.1' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  -H "Content-Range: bytes 0-$((FILE_SIZE - 1))/*" \
+  --data-binary @"$FILE"
+
+curl -s \
+  -X POST \
+  "${ACTIONS_CACHE_URL}_apis/artifactcache/caches/$CACHE_ID" \
+  -H 'Content-Type: application/json' \
+  -H 'Accept: application/json;api-version=6.0-preview.1' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  -d '{"size": '"$(stat --format="%s" $FILE)"'}'
+
+curl -s \
+  -X GET \
+  "${ACTIONS_CACHE_URL}_apis/artifactcache/cache?keys=$CACHE_KEY" \
+  -H 'Content-Type: application/json' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  | jq .
+"""
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.strings import camel_case
+from omlish.lite.strings import snake_case
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class GithubCacheServiceV1:
+    API_VERSION = '6.0-preview.1'
+
+    @classmethod
+    def get_service_url(cls, base_url: str) -> str:
+        return f'{base_url.rstrip("/")}/_apis/artifactcache'
+
+    #
+
+    @classmethod
+    def dataclass_to_json(cls, obj: ta.Any) -> ta.Any:
+        return {
+            camel_case(k, lower=True): v
+            for k, v in dc.asdict(obj).items()
+            if v is not None
+        }
+
+    @classmethod
+    def dataclass_from_json(cls, dcls: ta.Type[T], obj: ta.Any) -> T:
+        return dcls(**{
+            snake_case(k): v
+            for k, v in obj.items()
+        })
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class ArtifactCacheEntry:
+        cache_key: ta.Optional[str]
+        scope: ta.Optional[str]
+        cache_version: ta.Optional[str]
+        creation_time: ta.Optional[str]
+        archive_location: ta.Optional[str]
+
+    @dc.dataclass(frozen=True)
+    class ArtifactCacheList:
+        total_count: int
+        artifact_caches: ta.Optional[ta.Sequence['GithubCacheServiceV1.ArtifactCacheEntry']]
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class ReserveCacheRequest:
+        key: str
+        cache_size: ta.Optional[int]
+        version: ta.Optional[str] = None
+
+    @dc.dataclass(frozen=True)
+    class ReserveCacheResponse:
+        cache_id: int
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class CommitCacheRequest:
+        size: int
+
+    #
+
+    class CompressionMethod:
+        GZIP = 'gzip'
+        ZSTD_WITHOUT_LONG = 'zstd-without-long'
+        ZSTD = 'zstd'
+
+    @dc.dataclass(frozen=True)
+    class InternalCacheOptions:
+        compression_method: ta.Optional[str]  # CompressionMethod
+        enable_cross_os_archive: ta.Optional[bool]
+        cache_size: ta.Optional[int]
+
+
+class GithubCacheServiceV2:
+    SERVICE_NAME = 'github.actions.results.api.v1.CacheService'
+
+    @dc.dataclass(frozen=True)
+    class Method:
+        name: str
+        request: type
+        response: type
+
+    #
+
+    class CacheScopePermission:
+        READ = 1
+        WRITE = 2
+        ALL = READ | WRITE
+
+    @dc.dataclass(frozen=True)
+    class CacheScope:
+        scope: str
+        permission: int  # CacheScopePermission
+
+    @dc.dataclass(frozen=True)
+    class CacheMetadata:
+        repository_id: int
+        scope: ta.Sequence['GithubCacheServiceV2.CacheScope']
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class CreateCacheEntryRequest:
+        key: str
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+    @dc.dataclass(frozen=True)
+    class CreateCacheEntryResponse:
+        ok: bool
+        signed_upload_url: str
+
+    CREATE_CACHE_ENTRY_METHOD = Method(
+        'CreateCacheEntry',
+        CreateCacheEntryRequest,
+        CreateCacheEntryResponse,
+    )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class FinalizeCacheEntryUploadRequest:
+        key: str
+        size_bytes: int
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+    @dc.dataclass(frozen=True)
+    class FinalizeCacheEntryUploadResponse:
+        ok: bool
+        entry_id: str
+
+    FINALIZE_CACHE_ENTRY_METHOD = Method(
+        'FinalizeCacheEntryUpload',
+        FinalizeCacheEntryUploadRequest,
+        FinalizeCacheEntryUploadResponse,
+    )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class GetCacheEntryDownloadUrlRequest:
+        key: str
+        restore_keys: ta.Sequence[str]
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+    @dc.dataclass(frozen=True)
+    class GetCacheEntryDownloadUrlResponse:
+        ok: bool
+        signed_download_url: str
+        matched_key: str
+
+    GET_CACHE_ENTRY_DOWNLOAD_URL_METHOD = Method(
+        'GetCacheEntryDownloadURL',
+        GetCacheEntryDownloadUrlRequest,
+        GetCacheEntryDownloadUrlResponse,
+    )