omdev 0.0.0.dev212__py3-none-any.whl → 0.0.0.dev214__py3-none-any.whl

omdev/ci/github/client.py

@@ -0,0 +1,492 @@
+# ruff: noqa: TC003 UP006 UP007
+import abc
+import asyncio
+import dataclasses as dc
+import http.client
+import json
+import os
+import typing as ta
+import urllib.parse
+import urllib.request
+
+from omlish.asyncs.asyncio.asyncio import asyncio_wait_concurrent
+from omlish.lite.check import check
+from omlish.lite.json import json_dumps_compact
+from omlish.lite.logs import log
+
+from ..consts import CI_CACHE_VERSION
+from ..utils import log_timing_context
+from .api import GithubCacheServiceV1
+from .env import register_github_env_var
+
+
+##
+
+
+class GithubCacheClient(abc.ABC):
+    class Entry(abc.ABC):  # noqa
+        pass
+
+    @abc.abstractmethod
+    def get_entry(self, key: str) -> ta.Awaitable[ta.Optional[Entry]]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def download_file(self, entry: Entry, out_file: str) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def upload_file(self, key: str, in_file: str) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+
+##
+
+
+class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
+    BASE_URL_ENV_VAR = register_github_env_var('ACTIONS_CACHE_URL')
+    AUTH_TOKEN_ENV_VAR = register_github_env_var('ACTIONS_RUNTIME_TOKEN')  # noqa
+
+    KEY_SUFFIX_ENV_VAR = register_github_env_var('GITHUB_RUN_ID')
+
+    #
+
+    def __init__(
+            self,
+            *,
+            base_url: ta.Optional[str] = None,
+            auth_token: ta.Optional[str] = None,
+
+            key_prefix: ta.Optional[str] = None,
+            key_suffix: ta.Optional[str] = None,
+
+            cache_version: int = CI_CACHE_VERSION,
+
+            loop: ta.Optional[asyncio.AbstractEventLoop] = None,
+    ) -> None:
+        super().__init__()
+
+        #
+
+        if base_url is None:
+            base_url = check.non_empty_str(self.BASE_URL_ENV_VAR())
+        self._service_url = GithubCacheServiceV1.get_service_url(base_url)
+
+        if auth_token is None:
+            auth_token = self.AUTH_TOKEN_ENV_VAR()
+        self._auth_token = auth_token
+
+        #
+
+        self._key_prefix = key_prefix
+
+        if key_suffix is None:
+            key_suffix = self.KEY_SUFFIX_ENV_VAR()
+        self._key_suffix = check.non_empty_str(key_suffix)
+
+        #
+
+        self._cache_version = check.isinstance(cache_version, int)
+
+        #
+
+        self._given_loop = loop
+
+    #
+
+    def _get_loop(self) -> asyncio.AbstractEventLoop:
+        if (loop := self._given_loop) is not None:
+            return loop
+        return asyncio.get_event_loop()
+
+    #
+
+    def build_request_headers(
+            self,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            *,
+            content_type: ta.Optional[str] = None,
+            json_content: bool = False,
+    ) -> ta.Dict[str, str]:
+        dct = {
+            'Accept': ';'.join([
+                'application/json',
+                f'api-version={GithubCacheServiceV1.API_VERSION}',
+            ]),
+        }
+
+        if (auth_token := self._auth_token):
+            dct['Authorization'] = f'Bearer {auth_token}'
+
+        if content_type is None and json_content:
+            content_type = 'application/json'
+        if content_type is not None:
+            dct['Content-Type'] = content_type
+
+        if headers:
+            dct.update(headers)
+
+        return dct
+
+    #
+
+    def load_json_bytes(self, b: ta.Optional[bytes]) -> ta.Optional[ta.Any]:
+        if not b:
+            return None
+        return json.loads(b.decode('utf-8-sig'))
+
+    #
+
+    async def send_url_request(
+            self,
+            req: urllib.request.Request,
+    ) -> ta.Tuple[http.client.HTTPResponse, ta.Optional[bytes]]:
+        def run_sync():
+            with urllib.request.urlopen(req) as resp:  # noqa
+                body = resp.read()
+            return (resp, body)
+
+        return await self._get_loop().run_in_executor(None, run_sync)  # noqa
+
+    #
+
+    @dc.dataclass()
+    class ServiceRequestError(RuntimeError):
+        status_code: int
+        body: ta.Optional[bytes]
+
+        def __str__(self) -> str:
+            return repr(self)
+
+    async def send_service_request(
+            self,
+            path: str,
+            *,
+            method: ta.Optional[str] = None,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            content_type: ta.Optional[str] = None,
+            content: ta.Optional[bytes] = None,
+            json_content: ta.Optional[ta.Any] = None,
+            success_status_codes: ta.Optional[ta.Container[int]] = None,
+    ) -> ta.Optional[ta.Any]:
+        url = f'{self._service_url}/{path}'
+
+        if content is not None and json_content is not None:
+            raise RuntimeError('Must not pass both content and json_content')
+        elif json_content is not None:
+            content = json_dumps_compact(json_content).encode('utf-8')
+            header_json_content = True
+        else:
+            header_json_content = False
+
+        if method is None:
+            method = 'POST' if content is not None else 'GET'
+
+        #
+
+        req = urllib.request.Request(  # noqa
+            url,
+            method=method,
+            headers=self.build_request_headers(
+                headers,
+                content_type=content_type,
+                json_content=header_json_content,
+            ),
+            data=content,
+        )
+
+        resp, body = await self.send_url_request(req)
+
+        #
+
+        if success_status_codes is not None:
+            is_success = resp.status in success_status_codes
+        else:
+            is_success = (200 <= resp.status <= 300)
+        if not is_success:
+            raise self.ServiceRequestError(resp.status, body)
+
+        return self.load_json_bytes(body)
+
+    #
+
+    KEY_PART_SEPARATOR = '--'
+
+    def fix_key(self, s: str, partial_suffix: bool = False) -> str:
+        return self.KEY_PART_SEPARATOR.join([
+            *([self._key_prefix] if self._key_prefix else []),
+            s,
+            ('' if partial_suffix else self._key_suffix),
+        ])
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Entry(GithubCacheClient.Entry):
+        artifact: GithubCacheServiceV1.ArtifactCacheEntry
+
+    #
+
+    def build_get_entry_url_path(self, *keys: str) -> str:
+        qp = dict(
+            keys=','.join(urllib.parse.quote_plus(k) for k in keys),
+            version=str(self._cache_version),
+        )
+
+        return '?'.join([
+            'cache',
+            '&'.join([
+                f'{k}={v}'
+                for k, v in qp.items()
+            ]),
+        ])
+
+    GET_ENTRY_SUCCESS_STATUS_CODES = (200, 204)
+
+
+##
+
+
+class GithubCacheServiceV1Client(GithubCacheServiceV1BaseClient):
+    DEFAULT_CONCURRENCY = 4
+
+    DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024
+
+    def __init__(
+            self,
+            *,
+            concurrency: int = DEFAULT_CONCURRENCY,
+            chunk_size: int = DEFAULT_CHUNK_SIZE,
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(**kwargs)
+
+        check.arg(concurrency > 0)
+        self._concurrency = concurrency
+
+        check.arg(chunk_size > 0)
+        self._chunk_size = chunk_size
+
+    #
+
+    async def get_entry(self, key: str) -> ta.Optional[GithubCacheServiceV1BaseClient.Entry]:
+        obj = await self.send_service_request(
+            self.build_get_entry_url_path(self.fix_key(key, partial_suffix=True)),
+        )
+        if obj is None:
+            return None
+
+        return self.Entry(GithubCacheServiceV1.dataclass_from_json(
+            GithubCacheServiceV1.ArtifactCacheEntry,
+            obj,
+        ))
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class _DownloadChunk:
+        key: str
+        url: str
+        out_file: str
+        offset: int
+        size: int
+
+    async def _download_file_chunk_urllib(self, chunk: _DownloadChunk) -> None:
+        req = urllib.request.Request(  # noqa
+            chunk.url,
+            headers={
+                'Range': f'bytes={chunk.offset}-{chunk.offset + chunk.size - 1}',
+            },
+        )
+
+        _, buf_ = await self.send_url_request(req)
+
+        buf = check.not_none(buf_)
+        check.equal(len(buf), chunk.size)
+
+        #
+
+        def write_sync():
+            with open(chunk.out_file, 'r+b') as f:  # noqa
+                f.seek(chunk.offset, os.SEEK_SET)
+                f.write(buf)
+
+        await self._get_loop().run_in_executor(None, write_sync)  # noqa
+
+    # async def _download_file_chunk_curl(self, chunk: _DownloadChunk) -> None:
+    #     async with contextlib.AsyncExitStack() as es:
+    #         f = open(chunk.out_file, 'r+b')
+    #         f.seek(chunk.offset, os.SEEK_SET)
+    #
+    #         tmp_file = es.enter_context(temp_file_context())  # noqa
+    #
+    #         proc = await es.enter_async_context(asyncio_subprocesses.popen(
+    #             'curl',
+    #             '-s',
+    #             '-w', '%{json}',
+    #             '-H', f'Range: bytes={chunk.offset}-{chunk.offset + chunk.size - 1}',
+    #             chunk.url,
+    #             output=subprocess.PIPE,
+    #         ))
+    #
+    #         futs = asyncio.gather(
+    #
+    #         )
+    #
+    #         await proc.wait()
+    #
+    #         with open(tmp_file, 'r') as f:  # noqa
+    #             curl_json = tmp_file.read()
+    #
+    #     curl_res = json.loads(curl_json.decode().strip())
+    #
+    #     status_code = check.isinstance(curl_res['response_code'], int)
+    #
+    #     if not (200 <= status_code <= 300):
+    #         raise RuntimeError(f'Curl chunk download {chunk} failed: {curl_res}')
+
+    async def _download_file_chunk(self, chunk: _DownloadChunk) -> None:
+        with log_timing_context(
+                'Downloading github cache '
+                f'key {chunk.key} '
+                f'file {chunk.out_file} '
+                f'chunk {chunk.offset} - {chunk.offset + chunk.size}',
+        ):
+            await self._download_file_chunk_urllib(chunk)
+
+    async def _download_file(self, entry: GithubCacheServiceV1BaseClient.Entry, out_file: str) -> None:
+        key = check.non_empty_str(entry.artifact.cache_key)
+        url = check.non_empty_str(entry.artifact.archive_location)
+
+        head_resp, _ = await self.send_url_request(urllib.request.Request(  # noqa
+            url,
+            method='HEAD',
+        ))
+        file_size = int(head_resp.headers['Content-Length'])
+
+        #
+
+        with open(out_file, 'xb') as f:  # noqa
+            f.truncate(file_size)
+
+        #
+
+        download_tasks = []
+        chunk_size = self._chunk_size
+        for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
+            offset = i * chunk_size
+            size = min(chunk_size, file_size - offset)
+            chunk = self._DownloadChunk(
+                key,
+                url,
+                out_file,
+                offset,
+                size,
+            )
+            download_tasks.append(self._download_file_chunk(chunk))
+
+        await asyncio_wait_concurrent(download_tasks, self._concurrency)
+
+    async def download_file(self, entry: GithubCacheClient.Entry, out_file: str) -> None:
+        entry1 = check.isinstance(entry, self.Entry)
+        with log_timing_context(
+                'Downloading github cache '
+                f'key {entry1.artifact.cache_key} '
+                f'version {entry1.artifact.cache_version} '
+                f'to {out_file}',
+        ):
+            await self._download_file(entry1, out_file)
+
+    #
+
+    async def _upload_file_chunk(
+            self,
+            key: str,
+            cache_id: int,
+            in_file: str,
+            offset: int,
+            size: int,
+    ) -> None:
+        with log_timing_context(
+                f'Uploading github cache {key} '
+                f'file {in_file} '
+                f'chunk {offset} - {offset + size}',
+        ):
+            with open(in_file, 'rb') as f:  # noqa
+                f.seek(offset)
+                buf = f.read(size)
+
+            check.equal(len(buf), size)
+
+            await self.send_service_request(
+                f'caches/{cache_id}',
+                method='PATCH',
+                content_type='application/octet-stream',
+                headers={
+                    'Content-Range': f'bytes {offset}-{offset + size - 1}/*',
+                },
+                content=buf,
+                success_status_codes=[204],
+            )
+
+    async def _upload_file(self, key: str, in_file: str) -> None:
+        fixed_key = self.fix_key(key)
+
+        check.state(os.path.isfile(in_file))
+
+        file_size = os.stat(in_file).st_size
+
+        #
+
+        reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
+            key=fixed_key,
+            cache_size=file_size,
+            version=str(self._cache_version),
+        )
+        reserve_resp_obj = await self.send_service_request(
+            'caches',
+            json_content=GithubCacheServiceV1.dataclass_to_json(reserve_req),
+            success_status_codes=[201],
+        )
+        reserve_resp = GithubCacheServiceV1.dataclass_from_json(  # noqa
+            GithubCacheServiceV1.ReserveCacheResponse,
+            reserve_resp_obj,
+        )
+        cache_id = check.isinstance(reserve_resp.cache_id, int)
+
+        log.debug(f'Github cache file {os.path.basename(in_file)} got id {cache_id}')  # noqa
+
+        #
+
+        upload_tasks = []
+        chunk_size = self._chunk_size
+        for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
+            offset = i * chunk_size
+            size = min(chunk_size, file_size - offset)
+            upload_tasks.append(self._upload_file_chunk(
+                fixed_key,
+                cache_id,
+                in_file,
+                offset,
+                size,
+            ))
+
+        await asyncio_wait_concurrent(upload_tasks, self._concurrency)
+
+        #
+
+        commit_req = GithubCacheServiceV1.CommitCacheRequest(
+            size=file_size,
+        )
+        await self.send_service_request(
+            f'caches/{cache_id}',
+            json_content=GithubCacheServiceV1.dataclass_to_json(commit_req),
+            success_status_codes=[204],
+        )
+
+    async def upload_file(self, key: str, in_file: str) -> None:
+        with log_timing_context(
+                f'Uploading github cache file {os.path.basename(in_file)} '
+                f'key {key}',
+        ):
+            await self._upload_file(key, in_file)

omdev/ci/github/env.py

@@ -0,0 +1,21 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import os
+import typing as ta
+
+
+@dc.dataclass(frozen=True)
+class GithubEnvVar:
+    k: str
+
+    def __call__(self) -> ta.Optional[str]:
+        return os.environ.get(self.k)
+
+
+GITHUB_ENV_VARS: ta.Set[GithubEnvVar] = set()
+
+
+def register_github_env_var(k: str) -> GithubEnvVar:
+    ev = GithubEnvVar(k)
+    GITHUB_ENV_VARS.add(ev)
+    return ev

omdev/ci/requirements.py

@@ -1,5 +1,4 @@
 # ruff: noqa: UP006 UP007
-# @omlish-lite
 """
 TODO:
  - pip compile lol

omdev/ci/shell.py

@@ -1,5 +1,4 @@
 # ruff: noqa: UP006 UP007
-# @omlish-lite
 import dataclasses as dc
 import os
 import typing as ta

omdev/ci/utils.py

@@ -1,9 +1,6 @@
 # ruff: noqa: UP006 UP007
-# @omlish-lite
 import hashlib
 import logging
-import os.path
-import tempfile
 import time
 import typing as ta
 
@@ -13,15 +10,6 @@ from omlish.lite.logs import log
 ##
 
 
-def make_temp_file() -> str:
-    file_fd, file = tempfile.mkstemp()
-    os.close(file_fd)
-    return file
-
-
-##
-
-
 def read_yaml_file(yaml_file: str) -> ta.Any:
     yaml = __import__('yaml')
 
@@ -65,7 +53,7 @@ class LogTimingContext:
     def __enter__(self) -> 'LogTimingContext':
         self._begin_time = time.time()
 
-        self._log.log(self._level, f'Begin {self._description}')  # noqa
+        self._log.log(self._level, f'Begin : {self._description}')  # noqa
 
         return self
 
@@ -74,7 +62,7 @@ class LogTimingContext:
 
         self._log.log(
             self._level,
-            f'End {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
+            f'End : {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
         )
 
 

omdev/git/shallow.py

@@ -44,7 +44,7 @@ class GitShallowCloner:
                 'clone',
                 '-n',
                 '--depth=1',
-                *(['--filter=tree:0'] if self.repo_subtrees is not None else []),
+                '--filter=tree:0',
                 *(['-b', self.branch] if self.branch else []),
                 '--single-branch',
                 self.repo_url,