omdev 0.0.0.dev211__py3-none-any.whl → 0.0.0.dev213__py3-none-any.whl

omdev/ci/github/cache.py

@@ -1,14 +1,13 @@
 # ruff: noqa: UP006 UP007
 # @omlish-lite
+import abc
 import dataclasses as dc
-import json
 import os
 import shlex
 import typing as ta
+import urllib.parse
 
 from omlish.lite.check import check
-from omlish.lite.contextmanagers import defer
-from omlish.lite.json import json_dumps_compact
 from omlish.subprocesses import subprocesses
 
 from ..cache import DirectoryFileCache
@@ -16,256 +15,168 @@ from ..cache import ShellCache
 from ..shell import ShellCmd
 from ..utils import make_temp_file
 from .cacheapi import GithubCacheServiceV1
+from .curl import GithubServiceCurlClient
 
 
 ##
 
 
-class GithubV1CacheShellClient:
+class GithubCacheShellClient(abc.ABC):
+    class Entry(abc.ABC):  # noqa
+        pass
+
+    @abc.abstractmethod
+    def run_get_entry(self, key: str) -> ta.Optional[Entry]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def download_get_entry(self, entry: Entry, out_file: str) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def upload_cache_entry(self, key: str, in_file: str) -> None:
+        raise NotImplementedError
+
+
+#
+
+
+class GithubCacheServiceV1ShellClient(GithubCacheShellClient):
     BASE_URL_ENV_KEY = 'ACTIONS_CACHE_URL'
     AUTH_TOKEN_ENV_KEY = 'ACTIONS_RUNTIME_TOKEN'  # noqa
 
+    KEY_SUFFIX_ENV_KEY = 'GITHUB_RUN_ID'
+
+    CACHE_VERSION: ta.ClassVar[int] = 1
+
+    #
+
     def __init__(
             self,
             *,
             base_url: ta.Optional[str] = None,
             auth_token: ta.Optional[str] = None,
+
+            key_prefix: ta.Optional[str] = None,
+            key_suffix: ta.Optional[str] = None,
     ) -> None:
         super().__init__()
 
+        #
+
         if base_url is None:
             base_url = os.environ[self.BASE_URL_ENV_KEY]
-        self._base_url = check.non_empty_str(base_url)
+        service_url = GithubCacheServiceV1.get_service_url(base_url)
 
         if auth_token is None:
             auth_token = os.environ.get(self.AUTH_TOKEN_ENV_KEY)
-        self._auth_token = auth_token
-
-        self._service_url = GithubCacheServiceV1.get_service_url(self._base_url)
-
-    #
-
-    _MISSING = object()
 
-    def build_headers(
-            self,
-            *,
-            auth_token: ta.Any = _MISSING,
-            content_type: ta.Optional[str] = None,
-    ) -> ta.Dict[str, str]:
-        dct = {
-            'Accept': f'application/json;api-version={GithubCacheServiceV1.API_VERSION}',
-        }
+        self._curl = GithubServiceCurlClient(
+            service_url,
+            auth_token,
+            api_version=GithubCacheServiceV1.API_VERSION,
+        )
 
-        if auth_token is self._MISSING:
-            auth_token = self._auth_token
-        if auth_token:
-            dct['Authorization'] = f'Bearer {auth_token}'
+        #
 
-        if content_type is not None:
-            dct['Content-Type'] = content_type
+        self._key_prefix = key_prefix
 
-        return dct
+        if key_suffix is None:
+            key_suffix = os.environ[self.KEY_SUFFIX_ENV_KEY]
+        self._key_suffix = check.non_empty_str(key_suffix)
 
     #
 
-    HEADER_AUTH_TOKEN_ENV_KEY = '_GITHUB_CACHE_AUTH_TOKEN'  # noqa
-
-    def build_curl_cmd(
-            self,
-            method: str,
-            url: str,
-            *,
-            json_content: bool = False,
-            content_type: ta.Optional[str] = None,
-    ) -> ShellCmd:
-        if content_type is None and json_content:
-            content_type = 'application/json'
-
-        env = {}
-
-        header_auth_token: ta.Optional[str]
-        if self._auth_token:
-            env[self.HEADER_AUTH_TOKEN_ENV_KEY] = self._auth_token
-            header_auth_token = f'${self.HEADER_AUTH_TOKEN_ENV_KEY}'
-        else:
-            header_auth_token = None
-
-        hdrs = self.build_headers(
-            auth_token=header_auth_token,
-            content_type=content_type,
-        )
-
-        url = f'{self._service_url}/{url}'
+    KEY_PART_SEPARATOR = '--'
 
-        cmd = ' '.join([
-            'curl',
-            '-s',
-            '-X', method,
-            url,
-            *[f'-H "{k}: {v}"' for k, v in hdrs.items()],
+    def fix_key(self, s: str) -> str:
+        return self.KEY_PART_SEPARATOR.join([
+            *([self._key_prefix] if self._key_prefix else []),
+            s,
+            self._key_suffix,
         ])
 
-        return ShellCmd(
-            cmd,
-            env=env,
-        )
-
-    def build_post_json_curl_cmd(
-            self,
-            url: str,
-            obj: ta.Any,
-            **kwargs: ta.Any,
-    ) -> ShellCmd:
-        curl_cmd = self.build_curl_cmd(
-            'POST',
-            url,
-            json_content=True,
-            **kwargs,
-        )
-
-        obj_json = json_dumps_compact(obj)
-
-        return dc.replace(curl_cmd, s=f'{curl_cmd.s} -d {shlex.quote(obj_json)}')
-
     #
 
-    @dc.dataclass()
-    class CurlError(RuntimeError):
-        status_code: int
-        body: ta.Optional[bytes]
-
-        def __str__(self) -> str:
-            return repr(self)
-
     @dc.dataclass(frozen=True)
-    class CurlResult:
-        status_code: int
-        body: ta.Optional[bytes]
-
-        def as_error(self) -> 'GithubV1CacheShellClient.CurlError':
-            return GithubV1CacheShellClient.CurlError(
-                status_code=self.status_code,
-                body=self.body,
-            )
-
-    def run_curl_cmd(
-            self,
-            cmd: ShellCmd,
-            *,
-            raise_: bool = False,
-    ) -> CurlResult:
-        out_file = make_temp_file()
-        with defer(lambda: os.unlink(out_file)):
-            run_cmd = dc.replace(cmd, s=f"{cmd.s} -o {out_file} -w '%{{json}}'")
-
-            out_json_bytes = run_cmd.run(subprocesses.check_output)
-
-            out_json = json.loads(out_json_bytes.decode())
-            status_code = check.isinstance(out_json['response_code'], int)
-
-            with open(out_file, 'rb') as f:
-                body = f.read()
-
-            result = self.CurlResult(
-                status_code=status_code,
-                body=body,
-            )
-
-        if raise_ and (500 <= status_code <= 600):
-            raise result.as_error()
-
-        return result
-
-    def run_json_curl_cmd(
-            self,
-            cmd: ShellCmd,
-            *,
-            success_status_codes: ta.Optional[ta.Container[int]] = None,
-    ) -> ta.Optional[ta.Any]:
-        result = self.run_curl_cmd(cmd, raise_=True)
-
-        if success_status_codes is not None:
-            is_success = result.status_code in success_status_codes
-        else:
-            is_success = 200 <= result.status_code < 300
-
-        if is_success:
-            if not (body := result.body):
-                return None
-            return json.loads(body.decode('utf-8-sig'))
-
-        elif result.status_code == 404:
-            return None
-
-        else:
-            raise result.as_error()
+    class Entry(GithubCacheShellClient.Entry):
+        artifact: GithubCacheServiceV1.ArtifactCacheEntry
 
     #
 
     def build_get_entry_curl_cmd(self, key: str) -> ShellCmd:
-        return self.build_curl_cmd(
+        fixed_key = self.fix_key(key)
+
+        qp = dict(
+            keys=fixed_key,
+            version=str(self.CACHE_VERSION),
+        )
+
+        return self._curl.build_cmd(
             'GET',
-            f'cache?keys={key}',
+            shlex.quote('?'.join([
+                'cache',
+                '&'.join([
+                    f'{k}={urllib.parse.quote_plus(v)}'
+                    for k, v in qp.items()
+                ]),
+            ])),
         )
 
-    def run_get_entry(self, key: str) -> ta.Optional[GithubCacheServiceV1.ArtifactCacheEntry]:
-        curl_cmd = self.build_get_entry_curl_cmd(key)
+    def run_get_entry(self, key: str) -> ta.Optional[Entry]:
+        fixed_key = self.fix_key(key)
+        curl_cmd = self.build_get_entry_curl_cmd(fixed_key)
 
-        obj = self.run_json_curl_cmd(
+        obj = self._curl.run_json_cmd(
             curl_cmd,
             success_status_codes=[200, 204],
         )
         if obj is None:
             return None
 
-        return GithubCacheServiceV1.dataclass_from_json(
+        return self.Entry(GithubCacheServiceV1.dataclass_from_json(
             GithubCacheServiceV1.ArtifactCacheEntry,
             obj,
-        )
+        ))
 
     #
 
-    def build_download_get_entry_cmd(
-            self,
-            entry: GithubCacheServiceV1.ArtifactCacheEntry,
-            out_file: str,
-    ) -> ShellCmd:
+    def build_download_get_entry_cmd(self, entry: Entry, out_file: str) -> ShellCmd:
         return ShellCmd(' '.join([
             'aria2c',
             '-x', '4',
             '-o', out_file,
-            check.non_empty_str(entry.archive_location),
+            check.non_empty_str(entry.artifact.archive_location),
         ]))
 
-    def download_get_entry(
-            self,
-            entry: GithubCacheServiceV1.ArtifactCacheEntry,
-            out_file: str,
-    ) -> None:
-        dl_cmd = self.build_download_get_entry_cmd(entry, out_file)
+    def download_get_entry(self, entry: GithubCacheShellClient.Entry, out_file: str) -> None:
+        dl_cmd = self.build_download_get_entry_cmd(
+            check.isinstance(entry, GithubCacheServiceV1ShellClient.Entry),
+            out_file,
+        )
         dl_cmd.run(subprocesses.check_call)
 
     #
 
-    def upload_cache_entry(
-            self,
-            key: str,
-            in_file: str,
-    ) -> None:
+    def upload_cache_entry(self, key: str, in_file: str) -> None:
+        fixed_key = self.fix_key(key)
+
         check.state(os.path.isfile(in_file))
 
         file_size = os.stat(in_file).st_size
 
+        #
+
         reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
-            key=key,
+            key=fixed_key,
             cache_size=file_size,
+            version=str(self.CACHE_VERSION),
         )
-        reserve_cmd = self.build_post_json_curl_cmd(
+        reserve_cmd = self._curl.build_post_json_cmd(
             'caches',
             GithubCacheServiceV1.dataclass_to_json(reserve_req),
         )
-        reserve_resp_obj: ta.Any = check.not_none(self.run_json_curl_cmd(
+        reserve_resp_obj: ta.Any = check.not_none(self._curl.run_json_cmd(
             reserve_cmd,
             success_status_codes=[201],
         ))
@@ -273,8 +184,66 @@ class GithubV1CacheShellClient:
             GithubCacheServiceV1.ReserveCacheResponse,
             reserve_resp_obj,
         )
+        cache_id = check.isinstance(reserve_resp.cache_id, int)
+
+        #
+
+        tmp_file = make_temp_file()
+
+        print(f'{file_size=}')
+        num_written = 0
+        chunk_size = 32 * 1024 * 1024
+        for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
+            ofs = i * chunk_size
+            sz = min(chunk_size, file_size - ofs)
+
+            patch_cmd = self._curl.build_cmd(
+                'PATCH',
+                f'caches/{cache_id}',
+                content_type='application/octet-stream',
+                headers={
+                    'Content-Range': f'bytes {ofs}-{ofs + sz - 1}/*',
+                },
+            )
 
-        raise NotImplementedError
+            #
+
+            # patch_data_cmd = dc.replace(patch_cmd, s=' | '.join([
+            #     f'dd if={in_file} bs={chunk_size} skip={i} count=1 status=none',
+            #     f'{patch_cmd.s} --data-binary -',
+            # ]))
+            # print(f'{patch_data_cmd.s=}')
+            # patch_result = self._curl.run_cmd(patch_data_cmd, raise_=True)
+
+            #
+
+            with open(in_file, 'rb') as f:
+                f.seek(ofs)
+                buf = f.read(sz)
+            with open(tmp_file, 'wb') as f:
+                f.write(buf)
+            num_written += len(buf)
+            print(f'{num_written=}')
+            patch_data_cmd = dc.replace(patch_cmd, s=f'{patch_cmd.s} --data-binary @{tmp_file}')
+            print(f'{patch_data_cmd.s=}')
+            patch_result = self._curl.run_cmd(patch_data_cmd, raise_=True)
+
+            #
+
+            check.equal(patch_result.status_code, 204)
+            ofs += sz
+
+        #
+
+        commit_req = GithubCacheServiceV1.CommitCacheRequest(
+            size=file_size,
+        )
+        commit_cmd = self._curl.build_post_json_cmd(
+            f'caches/{cache_id}',
+            GithubCacheServiceV1.dataclass_to_json(commit_req),
+        )
+        commit_result = self._curl.run_cmd(commit_cmd, raise_=True)
+        check.equal(commit_result.status_code, 204)
 
 
 ##
@@ -285,15 +254,15 @@ class GithubShellCache(ShellCache):
             self,
             dir: str,  # noqa
             *,
-            client: ta.Optional[GithubV1CacheShellClient] = None,
+            client: ta.Optional[GithubCacheShellClient] = None,
     ) -> None:
         super().__init__()
 
         self._dir = check.not_none(dir)
 
         if client is None:
-            client = GithubV1CacheShellClient()
-        self._client = client
+            client = GithubCacheServiceV1ShellClient()
+        self._client: GithubCacheShellClient = client
 
         self._local = DirectoryFileCache(self._dir)
 

omdev/ci/github/cacheapi.py

@@ -93,7 +93,7 @@ class GithubCacheServiceV1:
     @dc.dataclass(frozen=True)
     class ReserveCacheRequest:
         key: str
-        cache_size: ta.Optional[int]
+        cache_size: ta.Optional[int] = None
         version: ta.Optional[str] = None
 
     @dc.dataclass(frozen=True)

omdev/ci/github/cli.py

@@ -11,7 +11,7 @@ from omlish.argparse.cli import argparse_arg
 from omlish.argparse.cli import argparse_cmd
 from omlish.lite.json import json_dumps_pretty
 
-from .cache import GithubV1CacheShellClient
+from .cache import GithubCacheServiceV1ShellClient
 
 
 class GithubCli(ArgparseCli):
@@ -19,7 +19,7 @@ class GithubCli(ArgparseCli):
         argparse_arg('key'),
     )
     def get_cache_entry(self) -> None:
-        shell_client = GithubV1CacheShellClient()
+        shell_client = GithubCacheServiceV1ShellClient()
         entry = shell_client.run_get_entry(self.args.key)
         if entry is None:
             return

omdev/ci/github/curl.py

@@ -0,0 +1,209 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import json
+import os
+import shlex
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import defer
+from omlish.lite.json import json_dumps_compact
+from omlish.subprocesses import subprocesses
+
+from ..shell import ShellCmd
+from ..utils import make_temp_file
+
+
+##
+
+
+class GithubServiceCurlClient:
+    def __init__(
+            self,
+            service_url: str,
+            auth_token: ta.Optional[str] = None,
+            *,
+            api_version: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+
+        self._service_url = check.non_empty_str(service_url)
+        self._auth_token = auth_token
+        self._api_version = api_version
+
+    #
+
+    _MISSING = object()
+
+    def build_headers(
+            self,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            *,
+            auth_token: ta.Any = _MISSING,
+            content_type: ta.Optional[str] = None,
+    ) -> ta.Dict[str, str]:
+        dct = {
+            'Accept': ';'.join([
+                'application/json',
+                *([f'api-version={self._api_version}'] if self._api_version else []),
+            ]),
+        }
+
+        if auth_token is self._MISSING:
+            auth_token = self._auth_token
+        if auth_token:
+            dct['Authorization'] = f'Bearer {auth_token}'
+
+        if content_type is not None:
+            dct['Content-Type'] = content_type
+
+        if headers:
+            dct.update(headers)
+
+        return dct
+
+    #
+
+    HEADER_AUTH_TOKEN_ENV_KEY_PREFIX = '_GITHUB_SERVICE_AUTH_TOKEN'  # noqa
+
+    @property
+    def header_auth_token_env_key(self) -> str:
+        return f'{self.HEADER_AUTH_TOKEN_ENV_KEY_PREFIX}_{id(self)}'
+
+    def build_cmd(
+            self,
+            method: str,
+            url: str,
+            *,
+            json_content: bool = False,
+            content_type: ta.Optional[str] = None,
+            headers: ta.Optional[ta.Dict[str, str]] = None,
+    ) -> ShellCmd:
+        if content_type is None and json_content:
+            content_type = 'application/json'
+
+        env = {}
+
+        header_auth_token: ta.Optional[str]
+        if self._auth_token:
+            header_env_key = self.header_auth_token_env_key
+            env[header_env_key] = self._auth_token
+            header_auth_token = f'${header_env_key}'
+        else:
+            header_auth_token = None
+
+        built_hdrs = self.build_headers(
+            headers,
+            auth_token=header_auth_token,
+            content_type=content_type,
+        )
+
+        url = f'{self._service_url}/{url}'
+
+        cmd = ' '.join([
+            'curl',
+            '-s',
+            '-X', method,
+            url,
+            *[f'-H "{k}: {v}"' for k, v in built_hdrs.items()],
+        ])
+
+        return ShellCmd(
+            cmd,
+            env=env,
+        )
+
+    def build_post_json_cmd(
+            self,
+            url: str,
+            obj: ta.Any,
+            **kwargs: ta.Any,
+    ) -> ShellCmd:
+        curl_cmd = self.build_cmd(
+            'POST',
+            url,
+            json_content=True,
+            **kwargs,
+        )
+
+        obj_json = json_dumps_compact(obj)
+
+        return dc.replace(curl_cmd, s=f'{curl_cmd.s} -d {shlex.quote(obj_json)}')
+
+    #
+
+    @dc.dataclass()
+    class Error(RuntimeError):
+        status_code: int
+        body: ta.Optional[bytes]
+
+        def __str__(self) -> str:
+            return repr(self)
+
+    @dc.dataclass(frozen=True)
+    class Result:
+        status_code: int
+        body: ta.Optional[bytes]
+
+        def as_error(self) -> 'GithubServiceCurlClient.Error':
+            return GithubServiceCurlClient.Error(
+                status_code=self.status_code,
+                body=self.body,
+            )
+
+    def run_cmd(
+            self,
+            cmd: ShellCmd,
+            *,
+            raise_: bool = False,
+            **subprocess_kwargs: ta.Any,
+    ) -> Result:
+        out_file = make_temp_file()
+        with defer(lambda: os.unlink(out_file)):
+            run_cmd = dc.replace(cmd, s=f"{cmd.s} -o {out_file} -w '%{{json}}'")
+
+            out_json_bytes = run_cmd.run(
+                subprocesses.check_output,
+                **subprocess_kwargs,
+            )
+
+            out_json = json.loads(out_json_bytes.decode())
+            status_code = check.isinstance(out_json['response_code'], int)
+
+            with open(out_file, 'rb') as f:
+                body = f.read()
+
+            result = self.Result(
+                status_code=status_code,
+                body=body,
+            )
+
+        if raise_ and (500 <= status_code <= 600):
+            raise result.as_error()
+
+        return result
+
+    def run_json_cmd(
+            self,
+            cmd: ShellCmd,
+            *,
+            success_status_codes: ta.Optional[ta.Container[int]] = None,
+    ) -> ta.Optional[ta.Any]:
+        result = self.run_cmd(cmd, raise_=True)
+
+        if success_status_codes is not None:
+            is_success = result.status_code in success_status_codes
+        else:
+            is_success = 200 <= result.status_code < 300
+
+        if is_success:
+            if not (body := result.body):
+                return None
+            return json.loads(body.decode('utf-8-sig'))
+
+        elif result.status_code == 404:
+            return None
+
+        else:
+            raise result.as_error()

omdev/ci/requirements.py

@@ -73,8 +73,8 @@ def download_requirements(
             'pip',
             'download',
             '-d', '/requirements',
-            *itertools.chain.from_iterable([
+            *itertools.chain.from_iterable(
                 ['-r', f'/requirements_txt/{os.path.basename(rt)}']
                 for rt in requirements_txts
-            ]),
+            ),
         )

omdev/git/shallow.py

@@ -44,7 +44,7 @@ class GitShallowCloner:
                 'clone',
                 '-n',
                 '--depth=1',
-                *(['--filter=tree:0'] if self.repo_subtrees is not None else []),
+                '--filter=tree:0',
                 *(['-b', self.branch] if self.branch else []),
                 '--single-branch',
                 self.repo_url,