omdev 0.0.0.dev209__py3-none-any.whl → 0.0.0.dev211__py3-none-any.whl

omdev/ci/github/cache.py

@@ -0,0 +1,355 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import json
+import os
+import shlex
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import defer
+from omlish.lite.json import json_dumps_compact
+from omlish.subprocesses import subprocesses
+
+from ..cache import DirectoryFileCache
+from ..cache import ShellCache
+from ..shell import ShellCmd
+from ..utils import make_temp_file
+from .cacheapi import GithubCacheServiceV1
+
+
+##
+
+
+class GithubV1CacheShellClient:
+    BASE_URL_ENV_KEY = 'ACTIONS_CACHE_URL'
+    AUTH_TOKEN_ENV_KEY = 'ACTIONS_RUNTIME_TOKEN'  # noqa
+
+    def __init__(
+            self,
+            *,
+            base_url: ta.Optional[str] = None,
+            auth_token: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+
+        if base_url is None:
+            base_url = os.environ[self.BASE_URL_ENV_KEY]
+        self._base_url = check.non_empty_str(base_url)
+
+        if auth_token is None:
+            auth_token = os.environ.get(self.AUTH_TOKEN_ENV_KEY)
+        self._auth_token = auth_token
+
+        self._service_url = GithubCacheServiceV1.get_service_url(self._base_url)
+
+    #
+
+    _MISSING = object()
+
+    def build_headers(
+            self,
+            *,
+            auth_token: ta.Any = _MISSING,
+            content_type: ta.Optional[str] = None,
+    ) -> ta.Dict[str, str]:
+        dct = {
+            'Accept': f'application/json;api-version={GithubCacheServiceV1.API_VERSION}',
+        }
+
+        if auth_token is self._MISSING:
+            auth_token = self._auth_token
+        if auth_token:
+            dct['Authorization'] = f'Bearer {auth_token}'
+
+        if content_type is not None:
+            dct['Content-Type'] = content_type
+
+        return dct
+
+    #
+
+    HEADER_AUTH_TOKEN_ENV_KEY = '_GITHUB_CACHE_AUTH_TOKEN'  # noqa
+
+    def build_curl_cmd(
+            self,
+            method: str,
+            url: str,
+            *,
+            json_content: bool = False,
+            content_type: ta.Optional[str] = None,
+    ) -> ShellCmd:
+        if content_type is None and json_content:
+            content_type = 'application/json'
+
+        env = {}
+
+        header_auth_token: ta.Optional[str]
+        if self._auth_token:
+            env[self.HEADER_AUTH_TOKEN_ENV_KEY] = self._auth_token
+            header_auth_token = f'${self.HEADER_AUTH_TOKEN_ENV_KEY}'
+        else:
+            header_auth_token = None
+
+        hdrs = self.build_headers(
+            auth_token=header_auth_token,
+            content_type=content_type,
+        )
+
+        url = f'{self._service_url}/{url}'
+
+        cmd = ' '.join([
+            'curl',
+            '-s',
+            '-X', method,
+            url,
+            *[f'-H "{k}: {v}"' for k, v in hdrs.items()],
+        ])
+
+        return ShellCmd(
+            cmd,
+            env=env,
+        )
+
+    def build_post_json_curl_cmd(
+            self,
+            url: str,
+            obj: ta.Any,
+            **kwargs: ta.Any,
+    ) -> ShellCmd:
+        curl_cmd = self.build_curl_cmd(
+            'POST',
+            url,
+            json_content=True,
+            **kwargs,
+        )
+
+        obj_json = json_dumps_compact(obj)
+
+        return dc.replace(curl_cmd, s=f'{curl_cmd.s} -d {shlex.quote(obj_json)}')
+
+    #
+
+    @dc.dataclass()
+    class CurlError(RuntimeError):
+        status_code: int
+        body: ta.Optional[bytes]
+
+        def __str__(self) -> str:
+            return repr(self)
+
+    @dc.dataclass(frozen=True)
+    class CurlResult:
+        status_code: int
+        body: ta.Optional[bytes]
+
+        def as_error(self) -> 'GithubV1CacheShellClient.CurlError':
+            return GithubV1CacheShellClient.CurlError(
+                status_code=self.status_code,
+                body=self.body,
+            )
+
+    def run_curl_cmd(
+            self,
+            cmd: ShellCmd,
+            *,
+            raise_: bool = False,
+    ) -> CurlResult:
+        out_file = make_temp_file()
+        with defer(lambda: os.unlink(out_file)):
+            run_cmd = dc.replace(cmd, s=f"{cmd.s} -o {out_file} -w '%{{json}}'")
+
+            out_json_bytes = run_cmd.run(subprocesses.check_output)
+
+            out_json = json.loads(out_json_bytes.decode())
+            status_code = check.isinstance(out_json['response_code'], int)
+
+            with open(out_file, 'rb') as f:
+                body = f.read()
+
+            result = self.CurlResult(
+                status_code=status_code,
+                body=body,
+            )
+
+        if raise_ and (500 <= status_code <= 600):
+            raise result.as_error()
+
+        return result
+
+    def run_json_curl_cmd(
+            self,
+            cmd: ShellCmd,
+            *,
+            success_status_codes: ta.Optional[ta.Container[int]] = None,
+    ) -> ta.Optional[ta.Any]:
+        result = self.run_curl_cmd(cmd, raise_=True)
+
+        if success_status_codes is not None:
+            is_success = result.status_code in success_status_codes
+        else:
+            is_success = 200 <= result.status_code < 300
+
+        if is_success:
+            if not (body := result.body):
+                return None
+            return json.loads(body.decode('utf-8-sig'))
+
+        elif result.status_code == 404:
+            return None
+
+        else:
+            raise result.as_error()
+
+    #
+
+    def build_get_entry_curl_cmd(self, key: str) -> ShellCmd:
+        return self.build_curl_cmd(
+            'GET',
+            f'cache?keys={key}',
+        )
+
+    def run_get_entry(self, key: str) -> ta.Optional[GithubCacheServiceV1.ArtifactCacheEntry]:
+        curl_cmd = self.build_get_entry_curl_cmd(key)
+
+        obj = self.run_json_curl_cmd(
+            curl_cmd,
+            success_status_codes=[200, 204],
+        )
+        if obj is None:
+            return None
+
+        return GithubCacheServiceV1.dataclass_from_json(
+            GithubCacheServiceV1.ArtifactCacheEntry,
+            obj,
+        )
+
+    #
+
+    def build_download_get_entry_cmd(
+            self,
+            entry: GithubCacheServiceV1.ArtifactCacheEntry,
+            out_file: str,
+    ) -> ShellCmd:
+        return ShellCmd(' '.join([
+            'aria2c',
+            '-x', '4',
+            '-o', out_file,
+            check.non_empty_str(entry.archive_location),
+        ]))
+
+    def download_get_entry(
+            self,
+            entry: GithubCacheServiceV1.ArtifactCacheEntry,
+            out_file: str,
+    ) -> None:
+        dl_cmd = self.build_download_get_entry_cmd(entry, out_file)
+        dl_cmd.run(subprocesses.check_call)
+
+    #
+
+    def upload_cache_entry(
+            self,
+            key: str,
+            in_file: str,
+    ) -> None:
+        check.state(os.path.isfile(in_file))
+
+        file_size = os.stat(in_file).st_size
+
+        reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
+            key=key,
+            cache_size=file_size,
+        )
+        reserve_cmd = self.build_post_json_curl_cmd(
+            'caches',
+            GithubCacheServiceV1.dataclass_to_json(reserve_req),
+        )
+        reserve_resp_obj: ta.Any = check.not_none(self.run_json_curl_cmd(
+            reserve_cmd,
+            success_status_codes=[201],
+        ))
+        reserve_resp = GithubCacheServiceV1.dataclass_from_json(  # noqa
+            GithubCacheServiceV1.ReserveCacheResponse,
+            reserve_resp_obj,
+        )
+
+        raise NotImplementedError
+
+
+##
+
+
+class GithubShellCache(ShellCache):
+    def __init__(
+            self,
+            dir: str,  # noqa
+            *,
+            client: ta.Optional[GithubV1CacheShellClient] = None,
+    ) -> None:
+        super().__init__()
+
+        self._dir = check.not_none(dir)
+
+        if client is None:
+            client = GithubV1CacheShellClient()
+        self._client = client
+
+        self._local = DirectoryFileCache(self._dir)
+
+    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+        local_file = self._local.get_cache_file_path(key)
+        if os.path.exists(local_file):
+            return ShellCmd(f'cat {shlex.quote(local_file)}')
+
+        if (entry := self._client.run_get_entry(key)) is None:
+            return None
+
+        tmp_file = self._local.format_incomplete_file(local_file)
+        try:
+            self._client.download_get_entry(entry, tmp_file)
+
+            os.replace(tmp_file, local_file)
+
+        except BaseException:  # noqa
+            os.unlink(tmp_file)
+
+            raise
+
+        return ShellCmd(f'cat {shlex.quote(local_file)}')
+
+    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
+        def __init__(
+                self,
+                owner: 'GithubShellCache',
+                key: str,
+                tmp_file: str,
+                local_file: str,
+        ) -> None:
+            super().__init__()
+
+            self._owner = owner
+            self._key = key
+            self._tmp_file = tmp_file
+            self._local_file = local_file
+
+        @property
+        def cmd(self) -> ShellCmd:
+            return ShellCmd(f'cat > {shlex.quote(self._tmp_file)}')
+
+        def _commit(self) -> None:
+            os.replace(self._tmp_file, self._local_file)
+
+            self._owner._client.upload_cache_entry(self._key, self._local_file)  # noqa
+
+        def _abort(self) -> None:
+            os.unlink(self._tmp_file)
+
+    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
+        local_file = self._local.get_cache_file_path(key, make_dirs=True)
+        return self._PutFileCmdContext(
+            self,
+            key,
+            self._local.format_incomplete_file(local_file),
+            local_file,
+        )

omdev/ci/github/cacheapi.py

@@ -0,0 +1,207 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+export FILE_SIZE=$(stat --format="%s" $FILE)
+
+export CACHE_ID=$(curl -s \
+  -X POST \
+  "${ACTIONS_CACHE_URL}_apis/artifactcache/caches" \
+  -H 'Content-Type: application/json' \
+  -H 'Accept: application/json;api-version=6.0-preview.1' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  -d '{"key": "'"$CACHE_KEY"'", "cacheSize": '"$FILE_SIZE"'}' \
+  | jq .cacheId)
+
+curl -s \
+  -X PATCH \
+  "${ACTIONS_CACHE_URL}_apis/artifactcache/caches/$CACHE_ID" \
+  -H 'Content-Type: application/octet-stream' \
+  -H 'Accept: application/json;api-version=6.0-preview.1' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  -H "Content-Range: bytes 0-$((FILE_SIZE - 1))/*" \
+  --data-binary @"$FILE"
+
+curl -s \
+  -X POST \
+  "${ACTIONS_CACHE_URL}_apis/artifactcache/caches/$CACHE_ID" \
+  -H 'Content-Type: application/json' \
+  -H 'Accept: application/json;api-version=6.0-preview.1' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  -d '{"size": '"$(stat --format="%s" $FILE)"'}'
+
+curl -s \
+  -X GET \
+  "${ACTIONS_CACHE_URL}_apis/artifactcache/cache?keys=$CACHE_KEY" \
+  -H 'Content-Type: application/json' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  | jq .
+"""
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.strings import camel_case
+from omlish.lite.strings import snake_case
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class GithubCacheServiceV1:
+    API_VERSION = '6.0-preview.1'
+
+    @classmethod
+    def get_service_url(cls, base_url: str) -> str:
+        return f'{base_url.rstrip("/")}/_apis/artifactcache'
+
+    #
+
+    @classmethod
+    def dataclass_to_json(cls, obj: ta.Any) -> ta.Any:
+        return {
+            camel_case(k, lower=True): v
+            for k, v in dc.asdict(obj).items()
+            if v is not None
+        }
+
+    @classmethod
+    def dataclass_from_json(cls, dcls: ta.Type[T], obj: ta.Any) -> T:
+        return dcls(**{
+            snake_case(k): v
+            for k, v in obj.items()
+        })
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class ArtifactCacheEntry:
+        cache_key: ta.Optional[str]
+        scope: ta.Optional[str]
+        cache_version: ta.Optional[str]
+        creation_time: ta.Optional[str]
+        archive_location: ta.Optional[str]
+
+    @dc.dataclass(frozen=True)
+    class ArtifactCacheList:
+        total_count: int
+        artifact_caches: ta.Optional[ta.Sequence['GithubCacheServiceV1.ArtifactCacheEntry']]
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class ReserveCacheRequest:
+        key: str
+        cache_size: ta.Optional[int]
+        version: ta.Optional[str] = None
+
+    @dc.dataclass(frozen=True)
+    class ReserveCacheResponse:
+        cache_id: int
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class CommitCacheRequest:
+        size: int
+
+    #
+
+    class CompressionMethod:
+        GZIP = 'gzip'
+        ZSTD_WITHOUT_LONG = 'zstd-without-long'
+        ZSTD = 'zstd'
+
+    @dc.dataclass(frozen=True)
+    class InternalCacheOptions:
+        compression_method: ta.Optional[str]  # CompressionMethod
+        enable_cross_os_archive: ta.Optional[bool]
+        cache_size: ta.Optional[int]
+
+
+class GithubCacheServiceV2:
+    SERVICE_NAME = 'github.actions.results.api.v1.CacheService'
+
+    @dc.dataclass(frozen=True)
+    class Method:
+        name: str
+        request: type
+        response: type
+
+    #
+
+    class CacheScopePermission:
+        READ = 1
+        WRITE = 2
+        ALL = READ | WRITE
+
+    @dc.dataclass(frozen=True)
+    class CacheScope:
+        scope: str
+        permission: int  # CacheScopePermission
+
+    @dc.dataclass(frozen=True)
+    class CacheMetadata:
+        repository_id: int
+        scope: ta.Sequence['GithubCacheServiceV2.CacheScope']
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class CreateCacheEntryRequest:
+        key: str
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+    @dc.dataclass(frozen=True)
+    class CreateCacheEntryResponse:
+        ok: bool
+        signed_upload_url: str
+
+    CREATE_CACHE_ENTRY_METHOD = Method(
+        'CreateCacheEntry',
+        CreateCacheEntryRequest,
+        CreateCacheEntryResponse,
+    )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class FinalizeCacheEntryUploadRequest:
+        key: str
+        size_bytes: int
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+    @dc.dataclass(frozen=True)
+    class FinalizeCacheEntryUploadResponse:
+        ok: bool
+        entry_id: str
+
+    FINALIZE_CACHE_ENTRY_METHOD = Method(
+        'FinalizeCacheEntryUpload',
+        FinalizeCacheEntryUploadRequest,
+        FinalizeCacheEntryUploadResponse,
+    )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class GetCacheEntryDownloadUrlRequest:
+        key: str
+        restore_keys: ta.Sequence[str]
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+    @dc.dataclass(frozen=True)
+    class GetCacheEntryDownloadUrlResponse:
+        ok: bool
+        signed_download_url: str
+        matched_key: str
+
+    GET_CACHE_ENTRY_DOWNLOAD_URL_METHOD = Method(
+        'GetCacheEntryDownloadURL',
+        GetCacheEntryDownloadUrlRequest,
+        GetCacheEntryDownloadUrlResponse,
+    )

omdev/ci/github/cli.py

@@ -0,0 +1,39 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+See:
+ - https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28
+"""
+import dataclasses as dc
+
+from omlish.argparse.cli import ArgparseCli
+from omlish.argparse.cli import argparse_arg
+from omlish.argparse.cli import argparse_cmd
+from omlish.lite.json import json_dumps_pretty
+
+from .cache import GithubV1CacheShellClient
+
+
+class GithubCli(ArgparseCli):
+    @argparse_cmd(
+        argparse_arg('key'),
+    )
+    def get_cache_entry(self) -> None:
+        shell_client = GithubV1CacheShellClient()
+        entry = shell_client.run_get_entry(self.args.key)
+        if entry is None:
+            return
+        print(json_dumps_pretty(dc.asdict(entry)))  # noqa
+
+    @argparse_cmd(
+        argparse_arg('repository-id'),
+    )
+    def list_cache_entries(self) -> None:
+        raise NotImplementedError
+
+
+if __name__ == '__main__':
+    def _main() -> None:
+        GithubCli().cli_run_and_exit()
+
+    _main()

omdev/ci/requirements.py

@@ -0,0 +1,80 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - pip compile lol
+  - but still support git+ stuff
+ - req.txt format aware hash
+  - more than just whitespace
+ - pyproject req rewriting
+ - download_requirements bootstrap off prev? not worth the dl?
+  - big deps (torch) change less, probably worth it
+ - follow embedded -r automatically like pyp
+"""
+import itertools
+import os.path
+import shutil
+import tempfile
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import defer
+from omlish.subprocesses import subprocesses
+
+from .utils import sha256_str
+
+
+##
+
+
+def build_requirements_hash(
+        requirements_txts: ta.Sequence[str],
+) -> str:
+    txt_file_contents: dict = {}
+
+    for txt_file in requirements_txts:
+        txt_file_name = os.path.basename(txt_file)
+        check.not_in(txt_file_name, txt_file_contents)
+        with open(txt_file) as f:
+            txt_contents = f.read()
+        txt_file_contents[txt_file_name] = txt_contents
+
+    #
+
+    lines = []
+    for txt_file, txt_contents in sorted(txt_file_contents.items()):
+        txt_hash = sha256_str(txt_contents)
+        lines.append(f'{txt_file}={txt_hash}')
+
+    return sha256_str('\n'.join(lines))
+
+
+##
+
+
+def download_requirements(
+        image: str,
+        requirements_dir: str,
+        requirements_txts: ta.Sequence[str],
+) -> None:
+    requirements_txt_dir = tempfile.mkdtemp()
+    with defer(lambda: shutil.rmtree(requirements_txt_dir)):
+        for rt in requirements_txts:
+            shutil.copyfile(rt, os.path.join(requirements_txt_dir, os.path.basename(rt)))
+
+        subprocesses.check_call(
+            'docker',
+            'run',
+            '--rm',
+            '-i',
+            '-v', f'{os.path.abspath(requirements_dir)}:/requirements',
+            '-v', f'{requirements_txt_dir}:/requirements_txt',
+            image,
+            'pip',
+            'download',
+            '-d', '/requirements',
+            *itertools.chain.from_iterable([
+                ['-r', f'/requirements_txt/{os.path.basename(rt)}']
+                for rt in requirements_txts
+            ]),
+        )