omdev 0.0.0.dev289__tar.gz → 0.0.0.dev290__tar.gz

{omdev-0.0.0.dev289/omdev.egg-info → omdev-0.0.0.dev290}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omdev
-Version: 0.0.0.dev289
+Version: 0.0.0.dev290
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev289
+Requires-Dist: omlish==0.0.0.dev290
 Provides-Extra: all
 Requires-Dist: black~=25.1; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

omdev-0.0.0.dev289/omdev/ci/github/client.py → omdev-0.0.0.dev290/omdev/ci/github/api/clients.py

@@ -1,8 +1,9 @@
-# ruff: noqa: TC003 UP006 UP007
+# ruff: noqa: UP006 UP007
 import abc
 import asyncio
 import dataclasses as dc
 import http.client
+import itertools
 import json
 import os
 import typing as ta
@@ -10,20 +11,21 @@ import urllib.parse
 import urllib.request
 
 from omlish.asyncs.asyncio.utils import asyncio_wait_concurrent
+from omlish.http.urllib import NonRaisingUrllibErrorProcessor
 from omlish.lite.check import check
 from omlish.lite.json import json_dumps_compact
 from omlish.lite.logs import log
 from omlish.lite.timing import log_timing_context
 
-from ..consts import CI_CACHE_VERSION
-from .api import GithubCacheServiceV1
-from .env import register_github_env_var
+from ...consts import CI_CACHE_VERSION
+from ..env import register_github_env_var
 
 
 ##
 
 
 class GithubCacheClient(abc.ABC):
+    @dc.dataclass(frozen=True)
     class Entry(abc.ABC):  # noqa
         pass
 
@@ -46,18 +48,21 @@ class GithubCacheClient(abc.ABC):
 ##
 
 
-class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
-    BASE_URL_ENV_VAR = register_github_env_var('ACTIONS_CACHE_URL')
+class BaseGithubCacheClient(GithubCacheClient, abc.ABC):
     AUTH_TOKEN_ENV_VAR = register_github_env_var('ACTIONS_RUNTIME_TOKEN')  # noqa
 
     KEY_SUFFIX_ENV_VAR = register_github_env_var('GITHUB_RUN_ID')
 
+    DEFAULT_CONCURRENCY = 4
+    DEFAULT_CHUNK_SIZE = 64 * 1024 * 1024
+
     #
 
     def __init__(
             self,
             *,
-            base_url: ta.Optional[str] = None,
+            service_url: str,
+
             auth_token: ta.Optional[str] = None,
 
             key_prefix: ta.Optional[str] = None,
@@ -66,14 +71,15 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
             cache_version: int = CI_CACHE_VERSION,
 
             loop: ta.Optional[asyncio.AbstractEventLoop] = None,
+
+            concurrency: int = DEFAULT_CONCURRENCY,
+            chunk_size: int = DEFAULT_CHUNK_SIZE,
     ) -> None:
         super().__init__()
 
         #
 
-        if base_url is None:
-            base_url = check.non_empty_str(self.BASE_URL_ENV_VAR())
-        self._service_url = GithubCacheServiceV1.get_service_url(base_url)
+        self._service_url = check.non_empty_str(service_url)
 
         if auth_token is None:
             auth_token = self.AUTH_TOKEN_ENV_VAR()
@@ -95,7 +101,16 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
 
         self._given_loop = loop
 
-    #
+        #
+
+        check.arg(concurrency > 0)
+        self._concurrency = concurrency
+
+        check.arg(chunk_size > 0)
+        self._chunk_size = chunk_size
+
+    ##
+    # misc
 
     def _get_loop(self) -> asyncio.AbstractEventLoop:
         if (loop := self._given_loop) is not None:
@@ -104,21 +119,25 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
 
     #
 
-    def build_request_headers(
+    def _load_json_bytes(self, b: ta.Optional[bytes]) -> ta.Optional[ta.Any]:
+        if not b:
+            return None
+        return json.loads(b.decode('utf-8-sig'))
+
+    ##
+    # requests
+
+    def _build_request_headers(
             self,
             headers: ta.Optional[ta.Mapping[str, str]] = None,
             *,
+            no_auth: bool = False,
             content_type: ta.Optional[str] = None,
             json_content: bool = False,
     ) -> ta.Dict[str, str]:
-        dct = {
-            'Accept': ';'.join([
-                'application/json',
-                f'api-version={GithubCacheServiceV1.API_VERSION}',
-            ]),
-        }
-
-        if (auth_token := self._auth_token):
+        dct = {}
+
+        if not no_auth and (auth_token := self._auth_token):
             dct['Authorization'] = f'Bearer {auth_token}'
 
         if content_type is None and json_content:
@@ -133,19 +152,13 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
 
     #
 
-    def load_json_bytes(self, b: ta.Optional[bytes]) -> ta.Optional[ta.Any]:
-        if not b:
-            return None
-        return json.loads(b.decode('utf-8-sig'))
-
-    #
-
-    async def send_url_request(
+    async def _send_urllib_request(
             self,
             req: urllib.request.Request,
     ) -> ta.Tuple[http.client.HTTPResponse, ta.Optional[bytes]]:
         def run_sync():
-            with urllib.request.urlopen(req) as resp:  # noqa
+            opener = urllib.request.build_opener(NonRaisingUrllibErrorProcessor)
+            with opener.open(req) as resp:  # noqa
                 body = resp.read()
             return (resp, body)
 
@@ -161,18 +174,32 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
         def __str__(self) -> str:
             return repr(self)
 
-    async def send_service_request(
+    async def _send_request(
             self,
-            path: str,
             *,
+            url: ta.Optional[str] = None,
+            path: ta.Optional[str] = None,
+
             method: ta.Optional[str] = None,
+
             headers: ta.Optional[ta.Mapping[str, str]] = None,
+            no_auth: bool = False,
             content_type: ta.Optional[str] = None,
+
             content: ta.Optional[bytes] = None,
             json_content: ta.Optional[ta.Any] = None,
+
             success_status_codes: ta.Optional[ta.Container[int]] = None,
+
+            retry_status_codes: ta.Optional[ta.Container[int]] = None,
+            num_retries: int = 0,
+            retry_sleep: ta.Optional[float] = None,
     ) -> ta.Optional[ta.Any]:
-        url = f'{self._service_url}/{path}'
+        if url is not None and path is not None:
+            raise RuntimeError('Must not pass both url and path')
+        elif path is not None:
+            url = f'{self._service_url}/{path}'
+        url = check.non_empty_str(url)
 
         if content is not None and json_content is not None:
             raise RuntimeError('Must not pass both content and json_content')
@@ -185,33 +212,52 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
         if method is None:
             method = 'POST' if content is not None else 'GET'
 
+        headers = self._build_request_headers(
+            headers,
+            no_auth=no_auth,
+            content_type=content_type,
+            json_content=header_json_content,
+        )
+
         #
 
-        req = urllib.request.Request(  # noqa
-            url,
-            method=method,
-            headers=self.build_request_headers(
-                headers,
-                content_type=content_type,
-                json_content=header_json_content,
-            ),
-            data=content,
-        )
+        for n in itertools.count():
+            req = urllib.request.Request(  # noqa
+                url,
+                method=method,
+                headers=headers,
+                data=content,
+            )
 
-        resp, body = await self.send_url_request(req)
+            resp, body = await self._send_urllib_request(req)
 
-        #
+            #
 
-        if success_status_codes is not None:
-            is_success = resp.status in success_status_codes
-        else:
-            is_success = (200 <= resp.status <= 300)
-        if not is_success:
-            raise self.ServiceRequestError(resp.status, body)
+            if success_status_codes is not None:
+                is_success = resp.status in success_status_codes
+            else:
+                is_success = (200 <= resp.status < 300)
+            if is_success:
+                return self._load_json_bytes(body)
 
-        return self.load_json_bytes(body)
+            #
 
-    #
+            log.debug(f'Request to url {url} got unsuccessful status code {resp.status}')  # noqa
+
+            if not (
+                retry_status_codes is not None and
+                resp.status in retry_status_codes and
+                n < num_retries
+            ):
+                raise self.ServiceRequestError(resp.status, body)
+
+            if retry_sleep is not None:
+                await asyncio.sleep(retry_sleep)
+
+        raise RuntimeError('Unreachable')
+
+    ##
+    # keys
 
     KEY_PART_SEPARATOR = '---'
 
@@ -222,73 +268,8 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
             ('' if partial_suffix else self._key_suffix),
         ])
 
-    #
-
-    @dc.dataclass(frozen=True)
-    class Entry(GithubCacheClient.Entry):
-        artifact: GithubCacheServiceV1.ArtifactCacheEntry
-
-    def get_entry_url(self, entry: GithubCacheClient.Entry) -> ta.Optional[str]:
-        entry1 = check.isinstance(entry, self.Entry)
-        return entry1.artifact.archive_location
-
-    #
-
-    def build_get_entry_url_path(self, *keys: str) -> str:
-        qp = dict(
-            keys=','.join(urllib.parse.quote_plus(k) for k in keys),
-            version=str(self._cache_version),
-        )
-
-        return '?'.join([
-            'cache',
-            '&'.join([
-                f'{k}={v}'
-                for k, v in qp.items()
-            ]),
-        ])
-
-    GET_ENTRY_SUCCESS_STATUS_CODES = (200, 204)
-
-
-##
-
-
-class GithubCacheServiceV1Client(GithubCacheServiceV1BaseClient):
-    DEFAULT_CONCURRENCY = 4
-
-    DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024
-
-    def __init__(
-            self,
-            *,
-            concurrency: int = DEFAULT_CONCURRENCY,
-            chunk_size: int = DEFAULT_CHUNK_SIZE,
-            **kwargs: ta.Any,
-    ) -> None:
-        super().__init__(**kwargs)
-
-        check.arg(concurrency > 0)
-        self._concurrency = concurrency
-
-        check.arg(chunk_size > 0)
-        self._chunk_size = chunk_size
-
-    #
-
-    async def get_entry(self, key: str) -> ta.Optional[GithubCacheServiceV1BaseClient.Entry]:
-        obj = await self.send_service_request(
-            self.build_get_entry_url_path(self.fix_key(key, partial_suffix=True)),
-        )
-        if obj is None:
-            return None
-
-        return self.Entry(GithubCacheServiceV1.dataclass_from_json(
-            GithubCacheServiceV1.ArtifactCacheEntry,
-            obj,
-        ))
-
-    #
+    ##
+    # downloading
 
     @dc.dataclass(frozen=True)
     class _DownloadChunk:
@@ -306,7 +287,7 @@ class GithubCacheServiceV1Client(GithubCacheServiceV1BaseClient):
             },
         )
 
-        _, buf_ = await self.send_url_request(req)
+        _, buf_ = await self._send_urllib_request(req)
 
         buf = check.not_none(buf_)
         check.equal(len(buf), chunk.size)
@@ -349,7 +330,7 @@ class GithubCacheServiceV1Client(GithubCacheServiceV1BaseClient):
     #
     #     status_code = check.isinstance(curl_res['response_code'], int)
     #
-    #     if not (200 <= status_code <= 300):
+    #     if not (200 <= status_code < 300):
     #         raise RuntimeError(f'Curl chunk download {chunk} failed: {curl_res}')
 
     async def _download_file_chunk(self, chunk: _DownloadChunk) -> None:
@@ -361,11 +342,17 @@ class GithubCacheServiceV1Client(GithubCacheServiceV1BaseClient):
         ):
             await self._download_file_chunk_urllib(chunk)
 
-    async def _download_file(self, entry: GithubCacheServiceV1BaseClient.Entry, out_file: str) -> None:
-        key = check.non_empty_str(entry.artifact.cache_key)
-        url = check.non_empty_str(entry.artifact.archive_location)
+    async def _download_file_chunks(
+            self,
+            *,
+            key: str,
+            url: str,
+            out_file: str,
+    ) -> None:
+        check.non_empty_str(key)
+        check.non_empty_str(url)
 
-        head_resp, _ = await self.send_url_request(urllib.request.Request(  # noqa
+        head_resp, _ = await self._send_urllib_request(urllib.request.Request(  # noqa
             url,
             method='HEAD',
         ))
@@ -394,74 +381,68 @@ class GithubCacheServiceV1Client(GithubCacheServiceV1BaseClient):
 
         await asyncio_wait_concurrent(download_tasks, self._concurrency)
 
-    async def download_file(self, entry: GithubCacheClient.Entry, out_file: str) -> None:
-        entry1 = check.isinstance(entry, self.Entry)
-        with log_timing_context(
-                'Downloading github cache '
-                f'key {entry1.artifact.cache_key} '
-                f'version {entry1.artifact.cache_version} '
-                f'to {out_file}',
-        ):
-            await self._download_file(entry1, out_file)
+    ##
+    # uploading
 
-    #
+    @dc.dataclass(frozen=True)
+    class _UploadChunk:
+        url: str
+        key: str
+        in_file: str
+        offset: int
+        size: int
 
-    async def _upload_file_chunk(
-            self,
-            key: str,
-            cache_id: int,
-            in_file: str,
-            offset: int,
-            size: int,
-    ) -> None:
-        with log_timing_context(
-                f'Uploading github cache {key} '
-                f'file {in_file} '
-                f'chunk {offset} - {offset + size}',
-        ):
-            with open(in_file, 'rb') as f:  # noqa
-                f.seek(offset)
-                buf = f.read(size)
-
-            check.equal(len(buf), size)
-
-            await self.send_service_request(
-                f'caches/{cache_id}',
-                method='PATCH',
-                content_type='application/octet-stream',
-                headers={
-                    'Content-Range': f'bytes {offset}-{offset + size - 1}/*',
-                },
-                content=buf,
-                success_status_codes=[204],
-            )
+    UPLOAD_CHUNK_NUM_RETRIES = 10
+    UPLOAD_CHUNK_RETRY_SLEEP = .5
 
-    async def _upload_file(self, key: str, in_file: str) -> None:
-        fixed_key = self.fix_key(key)
+    async def _upload_file_chunk_(self, chunk: _UploadChunk) -> None:
+        with open(chunk.in_file, 'rb') as f:  # noqa
+            f.seek(chunk.offset)
+            buf = f.read(chunk.size)
 
-        check.state(os.path.isfile(in_file))
+        check.equal(len(buf), chunk.size)
 
-        file_size = os.stat(in_file).st_size
+        await self._send_request(
+            url=chunk.url,
 
-        #
+            method='PATCH',
 
-        reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
-            key=fixed_key,
-            cache_size=file_size,
-            version=str(self._cache_version),
-        )
-        reserve_resp_obj = await self.send_service_request(
-            'caches',
-            json_content=GithubCacheServiceV1.dataclass_to_json(reserve_req),
-            success_status_codes=[201],
-        )
-        reserve_resp = GithubCacheServiceV1.dataclass_from_json(  # noqa
-            GithubCacheServiceV1.ReserveCacheResponse,
-            reserve_resp_obj,
+            headers={
+                'Content-Range': f'bytes {chunk.offset}-{chunk.offset + chunk.size - 1}/*',
+            },
+            no_auth=True,
+            content_type='application/octet-stream',
+
+            content=buf,
+
+            success_status_codes=[204],
+
+            # retry_status_codes=[405],
+            num_retries=self.UPLOAD_CHUNK_NUM_RETRIES,
+            retry_sleep=self.UPLOAD_CHUNK_RETRY_SLEEP,
         )
-        cache_id = check.isinstance(reserve_resp.cache_id, int)
 
-        log.debug(f'Github cache file {os.path.basename(in_file)} got id {cache_id}')  # noqa
+    async def _upload_file_chunk(self, chunk: _UploadChunk) -> None:
+        with log_timing_context(
+                f'Uploading github cache {chunk.key} '
+                f'file {chunk.in_file} '
+                f'chunk {chunk.offset} - {chunk.offset + chunk.size}',
+        ):
+            await self._upload_file_chunk_(chunk)
+
+    async def _upload_file_chunks(
+            self,
+            *,
+            in_file: str,
+            url: str,
+            key: str,
+
+            file_size: ta.Optional[int] = None,
+    ) -> None:
+        check.state(os.path.isfile(in_file))
+
+        if file_size is None:
+            file_size = os.stat(in_file).st_size
 
         #
 
@@ -470,30 +451,12 @@ class GithubCacheServiceV1Client(GithubCacheServiceV1BaseClient):
         for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
             offset = i * chunk_size
             size = min(chunk_size, file_size - offset)
-            upload_tasks.append(self._upload_file_chunk(
-                fixed_key,
-                cache_id,
-                in_file,
-                offset,
-                size,
-            ))
+            upload_tasks.append(self._upload_file_chunk(self._UploadChunk(
+                url=url,
+                key=key,
+                in_file=in_file,
+                offset=offset,
+                size=size,
+            )))
 
         await asyncio_wait_concurrent(upload_tasks, self._concurrency)
-
-        #
-
-        commit_req = GithubCacheServiceV1.CommitCacheRequest(
-            size=file_size,
-        )
-        await self.send_service_request(
-            f'caches/{cache_id}',
-            json_content=GithubCacheServiceV1.dataclass_to_json(commit_req),
-            success_status_codes=[204],
-        )
-
-    async def upload_file(self, key: str, in_file: str) -> None:
-        with log_timing_context(
-                f'Uploading github cache file {os.path.basename(in_file)} '
-                f'key {key}',
-        ):
-            await self._upload_file(key, in_file)

{omdev-0.0.0.dev289/omdev/ci/github → omdev-0.0.0.dev290/omdev/ci/github/api/v1}/api.py

@@ -49,6 +49,11 @@ T = ta.TypeVar('T')
 
 
 class GithubCacheServiceV1:
+    def __new__(cls, *args, **kwargs):  # noqa
+        raise TypeError
+
+    #
+
     API_VERSION = '6.0-preview.1'
 
     @classmethod
@@ -117,90 +122,3 @@ class GithubCacheServiceV1:
         compression_method: ta.Optional[str]  # CompressionMethod
         enable_cross_os_archive: ta.Optional[bool]
         cache_size: ta.Optional[int]
-
-
-class GithubCacheServiceV2:
-    SERVICE_NAME = 'github.actions.results.api.v1.CacheService'
-
-    @dc.dataclass(frozen=True)
-    class Method:
-        name: str
-        request: type
-        response: type
-
-    #
-
-    class CacheScopePermission:
-        READ = 1
-        WRITE = 2
-        ALL = READ | WRITE
-
-    @dc.dataclass(frozen=True)
-    class CacheScope:
-        scope: str
-        permission: int  # CacheScopePermission
-
-    @dc.dataclass(frozen=True)
-    class CacheMetadata:
-        repository_id: int
-        scope: ta.Sequence['GithubCacheServiceV2.CacheScope']
-
-    #
-
-    @dc.dataclass(frozen=True)
-    class CreateCacheEntryRequest:
-        key: str
-        version: str
-        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
-
-    @dc.dataclass(frozen=True)
-    class CreateCacheEntryResponse:
-        ok: bool
-        signed_upload_url: str
-
-    CREATE_CACHE_ENTRY_METHOD = Method(
-        'CreateCacheEntry',
-        CreateCacheEntryRequest,
-        CreateCacheEntryResponse,
-    )
-
-    #
-
-    @dc.dataclass(frozen=True)
-    class FinalizeCacheEntryUploadRequest:
-        key: str
-        size_bytes: int
-        version: str
-        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
-
-    @dc.dataclass(frozen=True)
-    class FinalizeCacheEntryUploadResponse:
-        ok: bool
-        entry_id: str
-
-    FINALIZE_CACHE_ENTRY_METHOD = Method(
-        'FinalizeCacheEntryUpload',
-        FinalizeCacheEntryUploadRequest,
-        FinalizeCacheEntryUploadResponse,
-    )
-
-    #
-
-    @dc.dataclass(frozen=True)
-    class GetCacheEntryDownloadUrlRequest:
-        key: str
-        restore_keys: ta.Sequence[str]
-        version: str
-        metadata: ta.Optional['GithubCacheServiceV2.CacheMetadata'] = None
-
-    @dc.dataclass(frozen=True)
-    class GetCacheEntryDownloadUrlResponse:
-        ok: bool
-        signed_download_url: str
-        matched_key: str
-
-    GET_CACHE_ENTRY_DOWNLOAD_URL_METHOD = Method(
-        'GetCacheEntryDownloadURL',
-        GetCacheEntryDownloadUrlRequest,
-        GetCacheEntryDownloadUrlResponse,
-    )