omdev 0.0.0.dev288__py3-none-any.whl → 0.0.0.dev290__py3-none-any.whl

omdev/ci/github/api/v1/client.py

@@ -0,0 +1,171 @@
+# ruff: noqa: TC003 UP006 UP007
+import dataclasses as dc
+import os
+import typing as ta
+import urllib.parse
+import urllib.request
+
+from omlish.lite.check import check
+from omlish.lite.logs import log
+from omlish.lite.timing import log_timing_context
+
+from ...env import register_github_env_var
+from ..clients import BaseGithubCacheClient
+from ..clients import GithubCacheClient
+from .api import GithubCacheServiceV1
+
+
+##
+
+
+class GithubCacheServiceV1Client(BaseGithubCacheClient):
+    BASE_URL_ENV_VAR = register_github_env_var('ACTIONS_CACHE_URL')
+
+    def __init__(
+            self,
+            *,
+            base_url: ta.Optional[str] = None,
+
+            **kwargs: ta.Any,
+    ) -> None:
+        if base_url is None:
+            base_url = check.non_empty_str(self.BASE_URL_ENV_VAR())
+        service_url = GithubCacheServiceV1.get_service_url(base_url)
+
+        super().__init__(
+            service_url=service_url,
+            **kwargs,
+        )
+
+    #
+
+    def _build_request_headers(
+            self,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            **kwargs: ta.Any,
+    ) -> ta.Dict[str, str]:
+        return super()._build_request_headers(
+            {
+                'Accept': ';'.join([
+                    'application/json',
+                    f'api-version={GithubCacheServiceV1.API_VERSION}',
+                ]),
+                **(headers or {}),
+            },
+            **kwargs,
+        )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Entry(GithubCacheClient.Entry):
+        artifact: GithubCacheServiceV1.ArtifactCacheEntry
+
+    def get_entry_url(self, entry: GithubCacheClient.Entry) -> ta.Optional[str]:
+        entry1 = check.isinstance(entry, self.Entry)
+        return entry1.artifact.archive_location
+
+    #
+
+    def _build_get_entry_url_path(self, *keys: str) -> str:
+        qp = dict(
+            keys=','.join(urllib.parse.quote_plus(k) for k in keys),
+            version=str(self._cache_version),
+        )
+
+        return '?'.join([
+            'cache',
+            '&'.join([
+                f'{k}={v}'
+                for k, v in qp.items()
+            ]),
+        ])
+
+    GET_ENTRY_SUCCESS_STATUS_CODES = (200, 204)
+
+    #
+
+    async def get_entry(self, key: str) -> ta.Optional[GithubCacheClient.Entry]:
+        obj = await self._send_request(
+            path=self._build_get_entry_url_path(self.fix_key(key, partial_suffix=True)),
+        )
+        if obj is None:
+            return None
+
+        return self.Entry(GithubCacheServiceV1.dataclass_from_json(
+            GithubCacheServiceV1.ArtifactCacheEntry,
+            obj,
+        ))
+
+    #
+
+    async def download_file(self, entry: GithubCacheClient.Entry, out_file: str) -> None:
+        entry1 = check.isinstance(entry, self.Entry)
+        with log_timing_context(
+                'Downloading github cache '
+                f'key {entry1.artifact.cache_key} '
+                f'version {entry1.artifact.cache_version} '
+                f'to {out_file}',
+        ):
+            await self._download_file_chunks(
+                key=check.non_empty_str(entry1.artifact.cache_key),
+                url=check.non_empty_str(entry1.artifact.archive_location),
+                out_file=out_file,
+            )
+
+    #
+
+    async def _upload_file(self, key: str, in_file: str) -> None:
+        fixed_key = self.fix_key(key)
+
+        check.state(os.path.isfile(in_file))
+        file_size = os.stat(in_file).st_size
+
+        #
+
+        reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
+            key=fixed_key,
+            cache_size=file_size,
+            version=str(self._cache_version),
+        )
+        reserve_resp_obj = await self._send_request(
+            path='caches',
+            json_content=GithubCacheServiceV1.dataclass_to_json(reserve_req),
+            success_status_codes=[201],
+        )
+        reserve_resp = GithubCacheServiceV1.dataclass_from_json(  # noqa
+            GithubCacheServiceV1.ReserveCacheResponse,
+            reserve_resp_obj,
+        )
+        cache_id = check.isinstance(reserve_resp.cache_id, int)
+
+        log.debug(f'Github cache file {os.path.basename(in_file)} got id {cache_id}')  # noqa
+
+        #
+
+        url = f'{self._service_url}/caches/{cache_id}'
+
+        await self._upload_file_chunks(
+            in_file=in_file,
+            url=url,
+            key=fixed_key,
+            file_size=file_size,
+        )
+
+        #
+
+        commit_req = GithubCacheServiceV1.CommitCacheRequest(
+            size=file_size,
+        )
+        await self._send_request(
+            path=f'caches/{cache_id}',
+            json_content=GithubCacheServiceV1.dataclass_to_json(commit_req),
+            success_status_codes=[204],
+        )
+
+    async def upload_file(self, key: str, in_file: str) -> None:
+        with log_timing_context(
+                f'Uploading github cache file {os.path.basename(in_file)} '
+                f'key {key}',
+        ):
+            await self._upload_file(key, in_file)

omdev/ci/github/api/v2/api.py

@@ -0,0 +1,148 @@
+# ruff: noqa: UP006 UP007
+"""
+https://github.com/tonistiigi/go-actions-cache/blob/3e9a6642607fd6e4d5d4fdab7c91fe8bf4c36a25/cache_v2.go
+
+==
+
+curl -s \
+  -X POST \
+  "${ACTIONS_RESULTS_URL}twirp/github.actions.results.api.v1.CacheService/CreateCacheEntry" \
+  -H 'Content-Type: application/json' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  -d '{"key": "foo", "version": "0000000000000000000000000000000000000000000000000000000000000001" }' \
+  | jq .
+
+curl -s \
+  -X POST \
+  "${ACTIONS_RESULTS_URL}twirp/github.actions.results.api.v1.CacheService/GetCacheEntryDownloadURL" \
+  -H 'Content-Type: application/json' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  -d '{"key": "foo", "restoreKeys": [], "version": "0000000000000000000000000000000000000000000000000000000000000001" }' \
+  | jq .
+
+"""  # noqa
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.check import check
+
+
+T = ta.TypeVar('T')
+
+GithubCacheServiceV2RequestT = ta.TypeVar('GithubCacheServiceV2RequestT')
+GithubCacheServiceV2ResponseT = ta.TypeVar('GithubCacheServiceV2ResponseT')
+
+
+##
+
+
+class GithubCacheServiceV2:
+    def __new__(cls, *args, **kwargs):  # noqa
+        raise TypeError
+
+    #
+
+    SERVICE_NAME = 'github.actions.results.api.v1.CacheService'
+
+    @classmethod
+    def get_service_url(cls, base_url: str) -> str:
+        return f'{base_url.rstrip("/")}/twirp/{cls.SERVICE_NAME}'
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Method(ta.Generic[GithubCacheServiceV2RequestT, GithubCacheServiceV2ResponseT]):
+        name: str
+        request: ta.Type[GithubCacheServiceV2RequestT]
+        response: ta.Type[GithubCacheServiceV2ResponseT]
+
+    #
+
+    class CacheScopePermission:
+        READ = 1
+        WRITE = 2
+        ALL = READ | WRITE
+
+    @dc.dataclass(frozen=True)
+    class CacheScope:
+        scope: str
+        permission: int  # CacheScopePermission
+
+    @dc.dataclass(frozen=True)
+    class CacheMetadata:
+        repository_id: int
+        scope: ta.Sequence['GithubCacheServiceV2.CacheScope']
+
+    VERSION_LENGTH: int = 64
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class CreateCacheEntryRequest:
+        key: str
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+        def __post_init__(self) -> None:
+            check.equal(len(self.version), GithubCacheServiceV2.VERSION_LENGTH)
+
+    @dc.dataclass(frozen=True)
+    class CreateCacheEntryResponse:
+        ok: bool
+        signed_upload_url: str
+
+    CREATE_CACHE_ENTRY_METHOD: Method[
+        CreateCacheEntryRequest,
+        CreateCacheEntryResponse,
+    ] = Method(
+        'CreateCacheEntry',
+        CreateCacheEntryRequest,
+        CreateCacheEntryResponse,
+    )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class FinalizeCacheEntryUploadRequest:
+        key: str
+        size_bytes: int
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+    @dc.dataclass(frozen=True)
+    class FinalizeCacheEntryUploadResponse:
+        ok: bool
+        entry_id: str
+
+    FINALIZE_CACHE_ENTRY_METHOD: Method[
+        FinalizeCacheEntryUploadRequest,
+        FinalizeCacheEntryUploadResponse,
+    ] = Method(
+        'FinalizeCacheEntryUpload',
+        FinalizeCacheEntryUploadRequest,
+        FinalizeCacheEntryUploadResponse,
+    )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class GetCacheEntryDownloadUrlRequest:
+        key: str
+        restore_keys: ta.Sequence[str]
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+    @dc.dataclass(frozen=True)
+    class GetCacheEntryDownloadUrlResponse:
+        ok: bool
+        signed_download_url: str
+        matched_key: str
+
+    GET_CACHE_ENTRY_DOWNLOAD_URL_METHOD: Method[
+        GetCacheEntryDownloadUrlRequest,
+        GetCacheEntryDownloadUrlResponse,
+    ] = Method(
+        'GetCacheEntryDownloadURL',
+        GetCacheEntryDownloadUrlRequest,
+        GetCacheEntryDownloadUrlResponse,
+    )

omdev/ci/github/api/v2/client.py

@@ -0,0 +1,166 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import os
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.logs import log
+from omlish.lite.timing import log_timing_context
+
+from ...env import register_github_env_var
+from ..clients import BaseGithubCacheClient
+from ..clients import GithubCacheClient
+from .api import GithubCacheServiceV2
+from .api import GithubCacheServiceV2RequestT
+from .api import GithubCacheServiceV2ResponseT
+
+
+##
+
+
+class GithubCacheServiceV2Client(BaseGithubCacheClient):
+    BASE_URL_ENV_VAR = register_github_env_var('ACTIONS_RESULTS_URL')
+
+    def __init__(
+            self,
+            *,
+            base_url: ta.Optional[str] = None,
+
+            **kwargs: ta.Any,
+    ) -> None:
+        if base_url is None:
+            base_url = check.non_empty_str(self.BASE_URL_ENV_VAR())
+        service_url = GithubCacheServiceV2.get_service_url(base_url)
+
+        super().__init__(
+            service_url=service_url,
+            **kwargs,
+        )
+
+    #
+
+    async def _send_method_request(
+            self,
+            method: GithubCacheServiceV2.Method[
+                GithubCacheServiceV2RequestT,
+                GithubCacheServiceV2ResponseT,
+            ],
+            request: GithubCacheServiceV2RequestT,
+            **kwargs: ta.Any,
+    ) -> ta.Optional[GithubCacheServiceV2ResponseT]:
+        obj = await self._send_request(
+            path=method.name,
+            json_content=dc.asdict(request),  # type: ignore[call-overload]
+            **kwargs,
+        )
+
+        if obj is None:
+            return None
+        return method.response(**obj)
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Entry(GithubCacheClient.Entry):
+        request: GithubCacheServiceV2.GetCacheEntryDownloadUrlRequest
+        response: GithubCacheServiceV2.GetCacheEntryDownloadUrlResponse
+
+        def __post_init__(self) -> None:
+            check.state(self.response.ok)
+            check.non_empty_str(self.response.signed_download_url)
+
+    def get_entry_url(self, entry: GithubCacheClient.Entry) -> ta.Optional[str]:
+        entry2 = check.isinstance(entry, self.Entry)
+        return check.non_empty_str(entry2.response.signed_download_url)
+
+    #
+
+    async def get_entry(self, key: str) -> ta.Optional[GithubCacheClient.Entry]:
+        version = str(self._cache_version).zfill(GithubCacheServiceV2.VERSION_LENGTH)
+
+        req = GithubCacheServiceV2.GetCacheEntryDownloadUrlRequest(
+            key=self.fix_key(key),
+            restore_keys=[self.fix_key(key, partial_suffix=True)],
+            version=version,
+        )
+
+        resp = await self._send_method_request(
+            GithubCacheServiceV2.GET_CACHE_ENTRY_DOWNLOAD_URL_METHOD,
+            req,
+        )
+        if resp is None or not resp.ok:
+            return None
+
+        return self.Entry(
+            request=req,
+            response=resp,
+        )
+
+    #
+
+    async def download_file(self, entry: GithubCacheClient.Entry, out_file: str) -> None:
+        entry2 = check.isinstance(entry, self.Entry)
+        with log_timing_context(
+                'Downloading github cache '
+                f'key {entry2.response.matched_key} '
+                f'version {entry2.request.version} '
+                f'to {out_file}',
+        ):
+            await self._download_file_chunks(
+                key=check.non_empty_str(entry2.response.matched_key),
+                url=check.non_empty_str(entry2.response.signed_download_url),
+                out_file=out_file,
+            )
+
+    #
+
+    async def _upload_file(self, key: str, in_file: str) -> None:
+        fixed_key = self.fix_key(key)
+
+        check.state(os.path.isfile(in_file))
+        file_size = os.stat(in_file).st_size
+
+        #
+
+        version = str(self._cache_version).zfill(GithubCacheServiceV2.VERSION_LENGTH)
+
+        reserve_resp = check.not_none(await self._send_method_request(
+            GithubCacheServiceV2.CREATE_CACHE_ENTRY_METHOD,  # type: ignore[arg-type]
+            GithubCacheServiceV2.CreateCacheEntryRequest(
+                key=fixed_key,
+                version=version,
+            ),
+        ))
+        check.state(reserve_resp.ok)
+
+        log.debug(f'Github cache file {os.path.basename(in_file)} upload reserved for file size {file_size}')  # noqa
+
+        #
+
+        await self._upload_file_chunks(
+            in_file=in_file,
+            url=reserve_resp.signed_upload_url,
+            key=fixed_key,
+            file_size=file_size,
+        )
+
+        #
+
+        commit_resp = check.not_none(await self._send_method_request(
+            GithubCacheServiceV2.FINALIZE_CACHE_ENTRY_METHOD,  # type: ignore[arg-type]
+            GithubCacheServiceV2.FinalizeCacheEntryUploadRequest(
+                key=key,
+                size_bytes=file_size,
+                version=version,
+            ),
+        ))
+        check.state(commit_resp.ok)
+
+        log.debug(f'Github cache file {os.path.basename(in_file)} upload complete, entry id {commit_resp.entry_id}')  # noqa
+
+    async def upload_file(self, key: str, in_file: str) -> None:
+        with log_timing_context(
+                f'Uploading github cache file {os.path.basename(in_file)} '
+                f'key {key}',
+        ):
+            await self._upload_file(key, in_file)

omdev/ci/github/cache.py

@@ -11,8 +11,9 @@ from ..cache import DataCache
 from ..cache import DirectoryFileCache
 from ..cache import FileCache
 from ..cache import FileCacheDataCache
-from .client import GithubCacheClient
-from .client import GithubCacheServiceV1Client
+from .api.clients import GithubCacheClient
+from .api.v1.client import GithubCacheServiceV1Client
+from .api.v2.client import GithubCacheServiceV2Client
 
 
 ##
@@ -23,11 +24,20 @@ class GithubCache(FileCache, DataCache):
     class Config:
         pass
 
+    DEFAULT_CLIENT_VERSION: ta.ClassVar[int] = 2
+
+    DEFAULT_CLIENTS_BY_VERSION: ta.ClassVar[ta.Mapping[int, ta.Callable[..., GithubCacheClient]]] = {
+        1: GithubCacheServiceV1Client,
+        2: GithubCacheServiceV2Client,
+    }
+
     def __init__(
             self,
             config: Config = Config(),
             *,
             client: ta.Optional[GithubCacheClient] = None,
+            default_client_version: ta.Optional[int] = None,
+
             version: ta.Optional[CacheVersion] = None,
 
             local: DirectoryFileCache,
@@ -39,7 +49,8 @@ class GithubCache(FileCache, DataCache):
         self._config = config
 
         if client is None:
-            client = GithubCacheServiceV1Client(
+            client_cls = self.DEFAULT_CLIENTS_BY_VERSION[default_client_version or self.DEFAULT_CLIENT_VERSION]
+            client = client_cls(
                 cache_version=self._version,
             )
         self._client: GithubCacheClient = client

omdev/ci/github/cli.py

@@ -10,7 +10,7 @@ from omlish.argparse.cli import argparse_arg
 from omlish.argparse.cli import argparse_cmd
 from omlish.lite.json import json_dumps_pretty
 
-from .client import GithubCacheServiceV1Client
+from .api.v1.client import GithubCacheServiceV1Client
 from .env import GITHUB_ENV_VARS
 
 

omdev/py/scripts/importtrace.py

@@ -488,20 +488,20 @@ def _main() -> None:
     import argparse
 
     parser = argparse.ArgumentParser()
-    parser.add_argument('--python')
+    parser.add_argument('-x', '--exe')
     parser.add_argument('--sqlite')
     parser.add_argument('--pretty', action='store_true')
     parser.add_argument('mod')
     args = parser.parse_args()
 
-    if args.python:
+    if args.exe:
         import inspect
         import subprocess
 
         mod_src = inspect.getsource(sys.modules[__name__])
         subprocess.run(
             [
-                args.python,
+                args.exe,
                 '-',
                 *(['--sqlite', args.sqlite] if args.sqlite else []),
                 *(['--pretty'] if args.pretty else []),