omdev 0.0.0.dev212__py3-none-any.whl → 0.0.0.dev214__py3-none-any.whl

omdev/ci/github/cache.py

@@ -1,355 +1,71 @@
 # ruff: noqa: UP006 UP007
-# @omlish-lite
-import dataclasses as dc
-import json
-import os
-import shlex
+import os.path
 import typing as ta
 
 from omlish.lite.check import check
-from omlish.lite.contextmanagers import defer
-from omlish.lite.json import json_dumps_compact
-from omlish.subprocesses import subprocesses
+from omlish.os.files import unlinking_if_exists
 
 from ..cache import DirectoryFileCache
-from ..cache import ShellCache
-from ..shell import ShellCmd
-from ..utils import make_temp_file
-from .cacheapi import GithubCacheServiceV1
+from ..cache import FileCache
+from .client import GithubCacheClient
+from .client import GithubCacheServiceV1Client
 
 
 ##
 
 
-class GithubV1CacheShellClient:
-    BASE_URL_ENV_KEY = 'ACTIONS_CACHE_URL'
-    AUTH_TOKEN_ENV_KEY = 'ACTIONS_RUNTIME_TOKEN'  # noqa
-
-    def __init__(
-            self,
-            *,
-            base_url: ta.Optional[str] = None,
-            auth_token: ta.Optional[str] = None,
-    ) -> None:
-        super().__init__()
-
-        if base_url is None:
-            base_url = os.environ[self.BASE_URL_ENV_KEY]
-        self._base_url = check.non_empty_str(base_url)
-
-        if auth_token is None:
-            auth_token = os.environ.get(self.AUTH_TOKEN_ENV_KEY)
-        self._auth_token = auth_token
-
-        self._service_url = GithubCacheServiceV1.get_service_url(self._base_url)
-
-    #
-
-    _MISSING = object()
-
-    def build_headers(
-            self,
-            *,
-            auth_token: ta.Any = _MISSING,
-            content_type: ta.Optional[str] = None,
-    ) -> ta.Dict[str, str]:
-        dct = {
-            'Accept': f'application/json;api-version={GithubCacheServiceV1.API_VERSION}',
-        }
-
-        if auth_token is self._MISSING:
-            auth_token = self._auth_token
-        if auth_token:
-            dct['Authorization'] = f'Bearer {auth_token}'
-
-        if content_type is not None:
-            dct['Content-Type'] = content_type
-
-        return dct
-
-    #
-
-    HEADER_AUTH_TOKEN_ENV_KEY = '_GITHUB_CACHE_AUTH_TOKEN'  # noqa
-
-    def build_curl_cmd(
-            self,
-            method: str,
-            url: str,
-            *,
-            json_content: bool = False,
-            content_type: ta.Optional[str] = None,
-    ) -> ShellCmd:
-        if content_type is None and json_content:
-            content_type = 'application/json'
-
-        env = {}
-
-        header_auth_token: ta.Optional[str]
-        if self._auth_token:
-            env[self.HEADER_AUTH_TOKEN_ENV_KEY] = self._auth_token
-            header_auth_token = f'${self.HEADER_AUTH_TOKEN_ENV_KEY}'
-        else:
-            header_auth_token = None
-
-        hdrs = self.build_headers(
-            auth_token=header_auth_token,
-            content_type=content_type,
-        )
-
-        url = f'{self._service_url}/{url}'
-
-        cmd = ' '.join([
-            'curl',
-            '-s',
-            '-X', method,
-            url,
-            *[f'-H "{k}: {v}"' for k, v in hdrs.items()],
-        ])
-
-        return ShellCmd(
-            cmd,
-            env=env,
-        )
-
-    def build_post_json_curl_cmd(
-            self,
-            url: str,
-            obj: ta.Any,
-            **kwargs: ta.Any,
-    ) -> ShellCmd:
-        curl_cmd = self.build_curl_cmd(
-            'POST',
-            url,
-            json_content=True,
-            **kwargs,
-        )
-
-        obj_json = json_dumps_compact(obj)
-
-        return dc.replace(curl_cmd, s=f'{curl_cmd.s} -d {shlex.quote(obj_json)}')
-
-    #
-
-    @dc.dataclass()
-    class CurlError(RuntimeError):
-        status_code: int
-        body: ta.Optional[bytes]
-
-        def __str__(self) -> str:
-            return repr(self)
-
-    @dc.dataclass(frozen=True)
-    class CurlResult:
-        status_code: int
-        body: ta.Optional[bytes]
-
-        def as_error(self) -> 'GithubV1CacheShellClient.CurlError':
-            return GithubV1CacheShellClient.CurlError(
-                status_code=self.status_code,
-                body=self.body,
-            )
-
-    def run_curl_cmd(
-            self,
-            cmd: ShellCmd,
-            *,
-            raise_: bool = False,
-    ) -> CurlResult:
-        out_file = make_temp_file()
-        with defer(lambda: os.unlink(out_file)):
-            run_cmd = dc.replace(cmd, s=f"{cmd.s} -o {out_file} -w '%{{json}}'")
-
-            out_json_bytes = run_cmd.run(subprocesses.check_output)
-
-            out_json = json.loads(out_json_bytes.decode())
-            status_code = check.isinstance(out_json['response_code'], int)
-
-            with open(out_file, 'rb') as f:
-                body = f.read()
-
-            result = self.CurlResult(
-                status_code=status_code,
-                body=body,
-            )
-
-        if raise_ and (500 <= status_code <= 600):
-            raise result.as_error()
-
-        return result
-
-    def run_json_curl_cmd(
-            self,
-            cmd: ShellCmd,
-            *,
-            success_status_codes: ta.Optional[ta.Container[int]] = None,
-    ) -> ta.Optional[ta.Any]:
-        result = self.run_curl_cmd(cmd, raise_=True)
-
-        if success_status_codes is not None:
-            is_success = result.status_code in success_status_codes
-        else:
-            is_success = 200 <= result.status_code < 300
-
-        if is_success:
-            if not (body := result.body):
-                return None
-            return json.loads(body.decode('utf-8-sig'))
-
-        elif result.status_code == 404:
-            return None
-
-        else:
-            raise result.as_error()
-
-    #
-
-    def build_get_entry_curl_cmd(self, key: str) -> ShellCmd:
-        return self.build_curl_cmd(
-            'GET',
-            f'cache?keys={key}',
-        )
-
-    def run_get_entry(self, key: str) -> ta.Optional[GithubCacheServiceV1.ArtifactCacheEntry]:
-        curl_cmd = self.build_get_entry_curl_cmd(key)
-
-        obj = self.run_json_curl_cmd(
-            curl_cmd,
-            success_status_codes=[200, 204],
-        )
-        if obj is None:
-            return None
-
-        return GithubCacheServiceV1.dataclass_from_json(
-            GithubCacheServiceV1.ArtifactCacheEntry,
-            obj,
-        )
-
-    #
-
-    def build_download_get_entry_cmd(
-            self,
-            entry: GithubCacheServiceV1.ArtifactCacheEntry,
-            out_file: str,
-    ) -> ShellCmd:
-        return ShellCmd(' '.join([
-            'aria2c',
-            '-x', '4',
-            '-o', out_file,
-            check.non_empty_str(entry.archive_location),
-        ]))
-
-    def download_get_entry(
-            self,
-            entry: GithubCacheServiceV1.ArtifactCacheEntry,
-            out_file: str,
-    ) -> None:
-        dl_cmd = self.build_download_get_entry_cmd(entry, out_file)
-        dl_cmd.run(subprocesses.check_call)
-
-    #
-
-    def upload_cache_entry(
-            self,
-            key: str,
-            in_file: str,
-    ) -> None:
-        check.state(os.path.isfile(in_file))
-
-        file_size = os.stat(in_file).st_size
-
-        reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
-            key=key,
-            cache_size=file_size,
-        )
-        reserve_cmd = self.build_post_json_curl_cmd(
-            'caches',
-            GithubCacheServiceV1.dataclass_to_json(reserve_req),
-        )
-        reserve_resp_obj: ta.Any = check.not_none(self.run_json_curl_cmd(
-            reserve_cmd,
-            success_status_codes=[201],
-        ))
-        reserve_resp = GithubCacheServiceV1.dataclass_from_json(  # noqa
-            GithubCacheServiceV1.ReserveCacheResponse,
-            reserve_resp_obj,
-        )
-
-        raise NotImplementedError
-
-
-##
-
-
-class GithubShellCache(ShellCache):
+class GithubFileCache(FileCache):
     def __init__(
             self,
             dir: str,  # noqa
             *,
-            client: ta.Optional[GithubV1CacheShellClient] = None,
+            client: ta.Optional[GithubCacheClient] = None,
+            **kwargs: ta.Any,
     ) -> None:
-        super().__init__()
+        super().__init__(**kwargs)
 
         self._dir = check.not_none(dir)
 
         if client is None:
-            client = GithubV1CacheShellClient()
-        self._client = client
+            client = GithubCacheServiceV1Client(
+                cache_version=self._version,
+            )
+        self._client: GithubCacheClient = client
 
-        self._local = DirectoryFileCache(self._dir)
+        self._local = DirectoryFileCache(
+            self._dir,
+            version=self._version,
+        )
 
-    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+    async def get_file(self, key: str) -> ta.Optional[str]:
         local_file = self._local.get_cache_file_path(key)
         if os.path.exists(local_file):
-            return ShellCmd(f'cat {shlex.quote(local_file)}')
+            return local_file
 
-        if (entry := self._client.run_get_entry(key)) is None:
+        if (entry := await self._client.get_entry(key)) is None:
             return None
 
         tmp_file = self._local.format_incomplete_file(local_file)
-        try:
-            self._client.download_get_entry(entry, tmp_file)
+        with unlinking_if_exists(tmp_file):
+            await self._client.download_file(entry, tmp_file)
 
             os.replace(tmp_file, local_file)
 
-        except BaseException:  # noqa
-            os.unlink(tmp_file)
-
-            raise
-
-        return ShellCmd(f'cat {shlex.quote(local_file)}')
-
-    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
-        def __init__(
-                self,
-                owner: 'GithubShellCache',
-                key: str,
-                tmp_file: str,
-                local_file: str,
-        ) -> None:
-            super().__init__()
-
-            self._owner = owner
-            self._key = key
-            self._tmp_file = tmp_file
-            self._local_file = local_file
-
-        @property
-        def cmd(self) -> ShellCmd:
-            return ShellCmd(f'cat > {shlex.quote(self._tmp_file)}')
-
-        def _commit(self) -> None:
-            os.replace(self._tmp_file, self._local_file)
+        return local_file
 
-            self._owner._client.upload_cache_entry(self._key, self._local_file)  # noqa
-
-        def _abort(self) -> None:
-            os.unlink(self._tmp_file)
-
-    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
-        local_file = self._local.get_cache_file_path(key, make_dirs=True)
-        return self._PutFileCmdContext(
+    async def put_file(
             self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = await self._local.put_file(
             key,
-            self._local.format_incomplete_file(local_file),
-            local_file,
+            file_path,
+            steal=steal,
         )
+
+        await self._client.upload_file(key, cache_file_path)
+
+        return cache_file_path

omdev/ci/github/cli.py

@@ -1,5 +1,4 @@
 # ruff: noqa: UP006 UP007
-# @omlish-lite
 """
 See:
  - https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28
@@ -11,16 +10,21 @@ from omlish.argparse.cli import argparse_arg
 from omlish.argparse.cli import argparse_cmd
 from omlish.lite.json import json_dumps_pretty
 
-from .cache import GithubV1CacheShellClient
+from .client import GithubCacheServiceV1Client
+from .env import GITHUB_ENV_VARS
 
 
 class GithubCli(ArgparseCli):
+    @argparse_cmd()
+    def list_referenced_env_vars(self) -> None:
+        print('\n'.join(sorted(ev.k for ev in GITHUB_ENV_VARS)))
+
     @argparse_cmd(
         argparse_arg('key'),
     )
-    def get_cache_entry(self) -> None:
-        shell_client = GithubV1CacheShellClient()
-        entry = shell_client.run_get_entry(self.args.key)
+    async def get_cache_entry(self) -> None:
+        client = GithubCacheServiceV1Client()
+        entry = await client.get_entry(self.args.key)
         if entry is None:
             return
         print(json_dumps_pretty(dc.asdict(entry)))  # noqa