omdev 0.0.0.dev289__py3-none-any.whl → 0.0.0.dev291__py3-none-any.whl

omdev/ci/github/api/v1/client.py

@@ -0,0 +1,171 @@
+# ruff: noqa: TC003 UP006 UP007
+import dataclasses as dc
+import os
+import typing as ta
+import urllib.parse
+import urllib.request
+
+from omlish.lite.check import check
+from omlish.lite.logs import log
+from omlish.lite.timing import log_timing_context
+
+from ...env import register_github_env_var
+from ..clients import BaseGithubCacheClient
+from ..clients import GithubCacheClient
+from .api import GithubCacheServiceV1
+
+
+##
+
+
+class GithubCacheServiceV1Client(BaseGithubCacheClient):
+    BASE_URL_ENV_VAR = register_github_env_var('ACTIONS_CACHE_URL')
+
+    def __init__(
+            self,
+            *,
+            base_url: ta.Optional[str] = None,
+
+            **kwargs: ta.Any,
+    ) -> None:
+        if base_url is None:
+            base_url = check.non_empty_str(self.BASE_URL_ENV_VAR())
+        service_url = GithubCacheServiceV1.get_service_url(base_url)
+
+        super().__init__(
+            service_url=service_url,
+            **kwargs,
+        )
+
+    #
+
+    def _build_request_headers(
+            self,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            **kwargs: ta.Any,
+    ) -> ta.Dict[str, str]:
+        return super()._build_request_headers(
+            {
+                'Accept': ';'.join([
+                    'application/json',
+                    f'api-version={GithubCacheServiceV1.API_VERSION}',
+                ]),
+                **(headers or {}),
+            },
+            **kwargs,
+        )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Entry(GithubCacheClient.Entry):
+        artifact: GithubCacheServiceV1.ArtifactCacheEntry
+
+    def get_entry_url(self, entry: GithubCacheClient.Entry) -> ta.Optional[str]:
+        entry1 = check.isinstance(entry, self.Entry)
+        return entry1.artifact.archive_location
+
+    #
+
+    def _build_get_entry_url_path(self, *keys: str) -> str:
+        qp = dict(
+            keys=','.join(urllib.parse.quote_plus(k) for k in keys),
+            version=str(self._cache_version),
+        )
+
+        return '?'.join([
+            'cache',
+            '&'.join([
+                f'{k}={v}'
+                for k, v in qp.items()
+            ]),
+        ])
+
+    GET_ENTRY_SUCCESS_STATUS_CODES = (200, 204)
+
+    #
+
+    async def get_entry(self, key: str) -> ta.Optional[GithubCacheClient.Entry]:
+        obj = await self._send_request(
+            path=self._build_get_entry_url_path(self.fix_key(key, partial_suffix=True)),
+        )
+        if obj is None:
+            return None
+
+        return self.Entry(GithubCacheServiceV1.dataclass_from_json(
+            GithubCacheServiceV1.ArtifactCacheEntry,
+            obj,
+        ))
+
+    #
+
+    async def download_file(self, entry: GithubCacheClient.Entry, out_file: str) -> None:
+        entry1 = check.isinstance(entry, self.Entry)
+        with log_timing_context(
+                'Downloading github cache '
+                f'key {entry1.artifact.cache_key} '
+                f'version {entry1.artifact.cache_version} '
+                f'to {out_file}',
+        ):
+            await self._download_file_chunks(
+                key=check.non_empty_str(entry1.artifact.cache_key),
+                url=check.non_empty_str(entry1.artifact.archive_location),
+                out_file=out_file,
+            )
+
+    #
+
+    async def _upload_file(self, key: str, in_file: str) -> None:
+        fixed_key = self.fix_key(key)
+
+        check.state(os.path.isfile(in_file))
+        file_size = os.stat(in_file).st_size
+
+        #
+
+        reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
+            key=fixed_key,
+            cache_size=file_size,
+            version=str(self._cache_version),
+        )
+        reserve_resp_obj = await self._send_request(
+            path='caches',
+            json_content=GithubCacheServiceV1.dataclass_to_json(reserve_req),
+            success_status_codes=[201],
+        )
+        reserve_resp = GithubCacheServiceV1.dataclass_from_json(  # noqa
+            GithubCacheServiceV1.ReserveCacheResponse,
+            reserve_resp_obj,
+        )
+        cache_id = check.isinstance(reserve_resp.cache_id, int)
+
+        log.debug(f'Github cache file {os.path.basename(in_file)} got id {cache_id}')  # noqa
+
+        #
+
+        url = f'{self._service_url}/caches/{cache_id}'
+
+        await self._upload_file_chunks(
+            in_file=in_file,
+            url=url,
+            key=fixed_key,
+            file_size=file_size,
+        )
+
+        #
+
+        commit_req = GithubCacheServiceV1.CommitCacheRequest(
+            size=file_size,
+        )
+        await self._send_request(
+            path=f'caches/{cache_id}',
+            json_content=GithubCacheServiceV1.dataclass_to_json(commit_req),
+            success_status_codes=[204],
+        )
+
+    async def upload_file(self, key: str, in_file: str) -> None:
+        with log_timing_context(
+                f'Uploading github cache file {os.path.basename(in_file)} '
+                f'key {key}',
+        ):
+            await self._upload_file(key, in_file)

omdev/ci/github/api/v2/api.py

@@ -0,0 +1,148 @@
+# ruff: noqa: UP006 UP007
+"""
+https://github.com/tonistiigi/go-actions-cache/blob/3e9a6642607fd6e4d5d4fdab7c91fe8bf4c36a25/cache_v2.go
+
+==
+
+curl -s \
+  -X POST \
+  "${ACTIONS_RESULTS_URL}twirp/github.actions.results.api.v1.CacheService/CreateCacheEntry" \
+  -H 'Content-Type: application/json' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  -d '{"key": "foo", "version": "0000000000000000000000000000000000000000000000000000000000000001" }' \
+  | jq .
+
+curl -s \
+  -X POST \
+  "${ACTIONS_RESULTS_URL}twirp/github.actions.results.api.v1.CacheService/GetCacheEntryDownloadURL" \
+  -H 'Content-Type: application/json' \
+  -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+  -d '{"key": "foo", "restoreKeys": [], "version": "0000000000000000000000000000000000000000000000000000000000000001" }' \
+  | jq .
+
+"""  # noqa
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.check import check
+
+
+T = ta.TypeVar('T')
+
+GithubCacheServiceV2RequestT = ta.TypeVar('GithubCacheServiceV2RequestT')
+GithubCacheServiceV2ResponseT = ta.TypeVar('GithubCacheServiceV2ResponseT')
+
+
+##
+
+
+class GithubCacheServiceV2:
+    def __new__(cls, *args, **kwargs):  # noqa
+        raise TypeError
+
+    #
+
+    SERVICE_NAME = 'github.actions.results.api.v1.CacheService'
+
+    @classmethod
+    def get_service_url(cls, base_url: str) -> str:
+        return f'{base_url.rstrip("/")}/twirp/{cls.SERVICE_NAME}'
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Method(ta.Generic[GithubCacheServiceV2RequestT, GithubCacheServiceV2ResponseT]):
+        name: str
+        request: ta.Type[GithubCacheServiceV2RequestT]
+        response: ta.Type[GithubCacheServiceV2ResponseT]
+
+    #
+
+    class CacheScopePermission:
+        READ = 1
+        WRITE = 2
+        ALL = READ | WRITE
+
+    @dc.dataclass(frozen=True)
+    class CacheScope:
+        scope: str
+        permission: int  # CacheScopePermission
+
+    @dc.dataclass(frozen=True)
+    class CacheMetadata:
+        repository_id: int
+        scope: ta.Sequence['GithubCacheServiceV2.CacheScope']
+
+    VERSION_LENGTH: int = 64
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class CreateCacheEntryRequest:
+        key: str
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+        def __post_init__(self) -> None:
+            check.equal(len(self.version), GithubCacheServiceV2.VERSION_LENGTH)
+
+    @dc.dataclass(frozen=True)
+    class CreateCacheEntryResponse:
+        ok: bool
+        signed_upload_url: str
+
+    CREATE_CACHE_ENTRY_METHOD: Method[
+        CreateCacheEntryRequest,
+        CreateCacheEntryResponse,
+    ] = Method(
+        'CreateCacheEntry',
+        CreateCacheEntryRequest,
+        CreateCacheEntryResponse,
+    )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class FinalizeCacheEntryUploadRequest:
+        key: str
+        size_bytes: int
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+    @dc.dataclass(frozen=True)
+    class FinalizeCacheEntryUploadResponse:
+        ok: bool
+        entry_id: str
+
+    FINALIZE_CACHE_ENTRY_METHOD: Method[
+        FinalizeCacheEntryUploadRequest,
+        FinalizeCacheEntryUploadResponse,
+    ] = Method(
+        'FinalizeCacheEntryUpload',
+        FinalizeCacheEntryUploadRequest,
+        FinalizeCacheEntryUploadResponse,
+    )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class GetCacheEntryDownloadUrlRequest:
+        key: str
+        restore_keys: ta.Sequence[str]
+        version: str
+        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
+
+    @dc.dataclass(frozen=True)
+    class GetCacheEntryDownloadUrlResponse:
+        ok: bool
+        signed_download_url: str
+        matched_key: str
+
+    GET_CACHE_ENTRY_DOWNLOAD_URL_METHOD: Method[
+        GetCacheEntryDownloadUrlRequest,
+        GetCacheEntryDownloadUrlResponse,
+    ] = Method(
+        'GetCacheEntryDownloadURL',
+        GetCacheEntryDownloadUrlRequest,
+        GetCacheEntryDownloadUrlResponse,
+    )

omdev/ci/github/api/v2/azure.py

@@ -0,0 +1,185 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - ominfra? no, circdep
+"""
+import base64
+import dataclasses as dc
+import datetime
+import typing as ta
+import urllib.parse
+import xml.etree.ElementTree as ET
+
+from omlish.asyncs.asyncio.utils import asyncio_wait_concurrent
+from omlish.lite.check import check
+from omlish.lite.logs import log
+from omlish.lite.timing import log_timing_context
+
+
+##
+
+
+class AzureBlockBlobUploader:
+    """
+    https://learn.microsoft.com/en-us/rest/api/storageservices/put-block
+    https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list
+    """
+
+    DEFAULT_CONCURRENCY = 4
+
+    @dc.dataclass(frozen=True)
+    class Request:
+        method: str
+        url: str
+        headers: ta.Optional[ta.Dict[str, str]] = None
+        body: ta.Optional[bytes] = None
+
+    @dc.dataclass(frozen=True)
+    class Response:
+        status: int
+        headers: ta.Optional[ta.Mapping[str, str]] = None
+        data: ta.Optional[bytes] = None
+
+        def get_header(self, name: str) -> ta.Optional[str]:
+            for k, v in (self.headers or {}).items():
+                if k.lower() == name.lower():
+                    return v
+            return None
+
+    def __init__(
+            self,
+            blob_url_with_sas: str,
+            make_request: ta.Callable[[Request], ta.Awaitable[Response]],
+            *,
+            api_version: str = '2020-10-02',
+            concurrency: int = DEFAULT_CONCURRENCY,
+    ) -> None:
+        """
+        blob_url_with_sas should be of the form:
+           https://<account>.blob.core.windows.net/<container>/<blob>?<SAS-token>
+        """
+
+        super().__init__()
+
+        self._make_request = make_request
+        self._api_version = api_version
+        check.arg(concurrency >= 1)
+        self._concurrency = concurrency
+
+        parsed = urllib.parse.urlparse(blob_url_with_sas)
+        self._base_url = f'{parsed.scheme}://{parsed.netloc}'
+        parts = parsed.path.lstrip('/').split('/', 1)
+        self._container = parts[0]
+        self._blob_name = parts[1]
+        self._sas = parsed.query
+
+    def _headers(self) -> ta.Dict[str, str]:
+        """Standard headers for Azure Blob REST calls."""
+
+        now = datetime.datetime.now(datetime.UTC).strftime('%a, %d %b %Y %H:%M:%S GMT')
+        return {
+            'x-ms-date': now,
+            'x-ms-version': self._api_version,
+        }
+
+    @dc.dataclass(frozen=True)
+    class FileChunk:
+        in_file: str
+        offset: int
+        size: int
+
+    async def _upload_file_chunk_(
+            self,
+            block_id: str,
+            chunk: FileChunk,
+    ) -> None:
+        with open(chunk.in_file, 'rb') as f:  # noqa
+            f.seek(chunk.offset)
+            data = f.read(chunk.size)
+
+        check.equal(len(data), chunk.size)
+
+        params = {
+            'comp': 'block',
+            'blockid': block_id,
+        }
+        query = self._sas + '&' + urllib.parse.urlencode(params)
+        url = f'{self._base_url}/{self._container}/{self._blob_name}?{query}'
+
+        log.debug(f'Uploading azure blob chunk {chunk} with block id {block_id}')  # noqa
+
+        resp = await self._make_request(self.Request(
+            'PUT',
+            url,
+            headers=self._headers(),
+            body=data,
+        ))
+        if resp.status not in (201, 202):
+            raise RuntimeError(f'Put Block failed: {block_id=} {resp.status=}')
+
+    async def _upload_file_chunk(
+            self,
+            block_id: str,
+            chunk: FileChunk,
+    ) -> None:
+        with log_timing_context(f'Uploading azure blob chunk {chunk} with block id {block_id}'):
+            await self._upload_file_chunk_(
+                block_id,
+                chunk,
+            )
+
+    async def upload_file(
+            self,
+            chunks: ta.List[FileChunk],
+    ) -> ta.Dict[str, ta.Any]:
+        block_ids = []
+
+        # 1) Stage each block
+        upload_tasks = []
+        for idx, chunk in enumerate(chunks):
+            # Generate a predictable block ID (must be URL-safe base64)
+            raw_id = f'{idx:08d}'.encode()
+            block_id = base64.b64encode(raw_id).decode('utf-8')
+            block_ids.append(block_id)
+
+            upload_tasks.append(self._upload_file_chunk(
+                block_id,
+                chunk,
+            ))
+
+        await asyncio_wait_concurrent(upload_tasks, self._concurrency)
+
+        # 2) Commit block list
+        root = ET.Element('BlockList')
+        for bid in block_ids:
+            elm = ET.SubElement(root, 'Latest')
+            elm.text = bid
+        body = ET.tostring(root, encoding='utf-8', method='xml')
+
+        params = {'comp': 'blocklist'}
+        query = self._sas + '&' + urllib.parse.urlencode(params)
+        url = f'{self._base_url}/{self._container}/{self._blob_name}?{query}'
+
+        log.debug(f'Putting azure blob chunk list block ids {block_ids}')  # noqa
+
+        resp = await self._make_request(self.Request(
+            'PUT',
+            url,
+            headers={
+                **self._headers(),
+                'Content-Type': 'application/xml',
+            },
+            body=body,
+        ))
+        if resp.status not in (200, 201):
+            raise RuntimeError(f'Put Block List failed: {resp.status} {resp.data!r}')
+
+        ret = {
+            'status_code': resp.status,
+            'etag': resp.get_header('ETag'),
+        }
+
+        log.debug(f'Uploaded azure blob chunk {ret}')  # noqa
+
+        return ret

omdev/ci/github/api/v2/client.py

@@ -0,0 +1,201 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import os
+import typing as ta
+import urllib.request
+
+from omlish.lite.check import check
+from omlish.lite.logs import log
+from omlish.lite.timing import log_timing_context
+
+from ...env import register_github_env_var
+from ..clients import BaseGithubCacheClient
+from ..clients import GithubCacheClient
+from .api import GithubCacheServiceV2
+from .api import GithubCacheServiceV2RequestT
+from .api import GithubCacheServiceV2ResponseT
+from .azure import AzureBlockBlobUploader
+
+
+##
+
+
+class GithubCacheServiceV2Client(BaseGithubCacheClient):
+    BASE_URL_ENV_VAR = register_github_env_var('ACTIONS_RESULTS_URL')
+
+    def __init__(
+            self,
+            *,
+            base_url: ta.Optional[str] = None,
+
+            **kwargs: ta.Any,
+    ) -> None:
+        if base_url is None:
+            base_url = check.non_empty_str(self.BASE_URL_ENV_VAR())
+        service_url = GithubCacheServiceV2.get_service_url(base_url)
+
+        super().__init__(
+            service_url=service_url,
+            **kwargs,
+        )
+
+    #
+
+    async def _send_method_request(
+            self,
+            method: GithubCacheServiceV2.Method[
+                GithubCacheServiceV2RequestT,
+                GithubCacheServiceV2ResponseT,
+            ],
+            request: GithubCacheServiceV2RequestT,
+            **kwargs: ta.Any,
+    ) -> ta.Optional[GithubCacheServiceV2ResponseT]:
+        obj = await self._send_request(
+            path=method.name,
+            json_content=dc.asdict(request),  # type: ignore[call-overload]
+            **kwargs,
+        )
+
+        if obj is None:
+            return None
+        return method.response(**obj)
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Entry(GithubCacheClient.Entry):
+        request: GithubCacheServiceV2.GetCacheEntryDownloadUrlRequest
+        response: GithubCacheServiceV2.GetCacheEntryDownloadUrlResponse
+
+        def __post_init__(self) -> None:
+            check.state(self.response.ok)
+            check.non_empty_str(self.response.signed_download_url)
+
+    def get_entry_url(self, entry: GithubCacheClient.Entry) -> ta.Optional[str]:
+        entry2 = check.isinstance(entry, self.Entry)
+        return check.non_empty_str(entry2.response.signed_download_url)
+
+    #
+
+    async def get_entry(self, key: str) -> ta.Optional[GithubCacheClient.Entry]:
+        version = str(self._cache_version).zfill(GithubCacheServiceV2.VERSION_LENGTH)
+
+        req = GithubCacheServiceV2.GetCacheEntryDownloadUrlRequest(
+            key=self.fix_key(key),
+            restore_keys=[self.fix_key(key, partial_suffix=True)],
+            version=version,
+        )
+
+        resp = await self._send_method_request(
+            GithubCacheServiceV2.GET_CACHE_ENTRY_DOWNLOAD_URL_METHOD,
+            req,
+        )
+        if resp is None or not resp.ok:
+            return None
+
+        return self.Entry(
+            request=req,
+            response=resp,
+        )
+
+    #
+
+    async def download_file(self, entry: GithubCacheClient.Entry, out_file: str) -> None:
+        entry2 = check.isinstance(entry, self.Entry)
+        with log_timing_context(
+                'Downloading github cache '
+                f'key {entry2.response.matched_key} '
+                f'version {entry2.request.version} '
+                f'to {out_file}',
+        ):
+            await self._download_file_chunks(
+                key=check.non_empty_str(entry2.response.matched_key),
+                url=check.non_empty_str(entry2.response.signed_download_url),
+                out_file=out_file,
+            )
+
+    #
+
+    async def _upload_file(self, key: str, in_file: str) -> None:
+        fixed_key = self.fix_key(key)
+
+        check.state(os.path.isfile(in_file))
+        file_size = os.stat(in_file).st_size
+
+        #
+
+        version = str(self._cache_version).zfill(GithubCacheServiceV2.VERSION_LENGTH)
+
+        reserve_resp = check.not_none(await self._send_method_request(
+            GithubCacheServiceV2.CREATE_CACHE_ENTRY_METHOD,  # type: ignore[arg-type]
+            GithubCacheServiceV2.CreateCacheEntryRequest(
+                key=fixed_key,
+                version=version,
+            ),
+        ))
+        check.state(reserve_resp.ok)
+
+        log.debug(f'Github cache file {os.path.basename(in_file)} upload reserved for file size {file_size}')  # noqa
+
+        #
+
+        upload_chunks = self._generate_file_upload_chunks(
+            in_file=in_file,
+            url=reserve_resp.signed_upload_url,
+            key=fixed_key,
+            file_size=file_size,
+        )
+
+        az_chunks = [
+            AzureBlockBlobUploader.FileChunk(
+                in_file=in_file,
+                offset=c.offset,
+                size=c.size,
+            )
+            for c in upload_chunks
+        ]
+
+        async def az_make_request(req: AzureBlockBlobUploader.Request) -> AzureBlockBlobUploader.Response:
+            u_req = urllib.request.Request(  # noqa
+                req.url,
+                method=req.method,
+                headers=req.headers or {},
+                data=req.body,
+            )
+
+            u_resp, u_body = await self._send_urllib_request(u_req)
+
+            return AzureBlockBlobUploader.Response(
+                status=u_resp.status,
+                headers=dict(u_resp.headers),
+                data=u_body,
+            )
+
+        az_uploader = AzureBlockBlobUploader(
+            reserve_resp.signed_upload_url,
+            az_make_request,
+            concurrency=self._concurrency,
+        )
+
+        await az_uploader.upload_file(az_chunks)
+
+        #
+
+        commit_resp = check.not_none(await self._send_method_request(
+            GithubCacheServiceV2.FINALIZE_CACHE_ENTRY_METHOD,  # type: ignore[arg-type]
+            GithubCacheServiceV2.FinalizeCacheEntryUploadRequest(
+                key=fixed_key,
+                size_bytes=file_size,
+                version=version,
+            ),
+        ))
+        check.state(commit_resp.ok)
+
+        log.debug(f'Github cache file {os.path.basename(in_file)} upload complete, entry id {commit_resp.entry_id}')  # noqa
+
+    async def upload_file(self, key: str, in_file: str) -> None:
+        with log_timing_context(
+                f'Uploading github cache file {os.path.basename(in_file)} '
+                f'key {key}',
+        ):
+            await self._upload_file(key, in_file)