omdev 0.0.0.dev223__py3-none-any.whl → 0.0.0.dev224__py3-none-any.whl

omdev/scripts/ci.py

@@ -159,6 +159,27 @@ class ShellCmd:
         )
 
 
+########################################
+# ../utils.py
+
+
+##
+
+
+def read_yaml_file(yaml_file: str) -> ta.Any:
+    yaml = __import__('yaml')
+
+    with open(yaml_file) as f:
+        return yaml.safe_load(f)
+
+
+##
+
+
+def sha256_str(s: str) -> str:
+    return hashlib.sha256(s.encode('utf-8')).hexdigest()
+
+
 ########################################
 # ../../../omlish/asyncs/asyncio/asyncio.py
 
@@ -1146,6 +1167,63 @@ class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
         self._underlying.handleError(record)
 
 
+########################################
+# ../../../omlish/logs/timing.py
+
+
+##
+
+
+class LogTimingContext:
+    DEFAULT_LOG: ta.ClassVar[ta.Optional[logging.Logger]] = None
+
+    class _NOT_SPECIFIED:  # noqa
+        def __new__(cls, *args, **kwargs):  # noqa
+            raise TypeError
+
+    def __init__(
+            self,
+            description: str,
+            *,
+            log: ta.Union[logging.Logger, ta.Type[_NOT_SPECIFIED], None] = _NOT_SPECIFIED,  # noqa
+            level: int = logging.DEBUG,
+    ) -> None:
+        super().__init__()
+
+        self._description = description
+        if log is self._NOT_SPECIFIED:
+            log = self.DEFAULT_LOG  # noqa
+        self._log: ta.Optional[logging.Logger] = log  # type: ignore
+        self._level = level
+
+    def set_description(self, description: str) -> 'LogTimingContext':
+        self._description = description
+        return self
+
+    _begin_time: float
+    _end_time: float
+
+    def __enter__(self) -> 'LogTimingContext':
+        self._begin_time = time.time()
+
+        if self._log is not None:
+            self._log.log(self._level, f'Begin : {self._description}')  # noqa
+
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._end_time = time.time()
+
+        if self._log is not None:
+            self._log.log(
+                self._level,
+                f'End : {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
+            )
+
+
+log_timing_context = LogTimingContext
+
+
 ########################################
 # ../../../omlish/os/files.py
 
@@ -1183,159 +1261,47 @@ def unlinking_if_exists(path: str) -> ta.Iterator[None]:
 
 
 ########################################
-# ../cache.py
-
-
-CacheVersion = ta.NewType('CacheVersion', int)
+# ../docker/utils.py
+"""
+TODO:
+ - some less stupid Dockerfile hash
+  - doesn't change too much though
+"""
 
 
 ##
 
 
-class FileCache(abc.ABC):
-    DEFAULT_CACHE_VERSION: ta.ClassVar[CacheVersion] = CacheVersion(CI_CACHE_VERSION)
-
-    def __init__(
-            self,
-            *,
-            version: ta.Optional[CacheVersion] = None,
-    ) -> None:
-        super().__init__()
-
-        if version is None:
-            version = self.DEFAULT_CACHE_VERSION
-        check.isinstance(version, int)
-        check.arg(version >= 0)
-        self._version: CacheVersion = version
-
-    @property
-    def version(self) -> CacheVersion:
-        return self._version
-
-    #
-
-    @abc.abstractmethod
-    def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def put_file(
-            self,
-            key: str,
-            file_path: str,
-            *,
-            steal: bool = False,
-    ) -> ta.Awaitable[str]:
-        raise NotImplementedError
-
-
-#
-
-
-class DirectoryFileCache(FileCache):
-    @dc.dataclass(frozen=True)
-    class Config:
-        dir: str
-
-        no_create: bool = False
-        no_purge: bool = False
-
-    def __init__(
-            self,
-            config: Config,
-            *,
-            version: ta.Optional[CacheVersion] = None,
-    ) -> None:  # noqa
-        super().__init__(
-            version=version,
-        )
-
-        self._config = config
-
-    @property
-    def dir(self) -> str:
-        return self._config.dir
-
-    #
-
-    VERSION_FILE_NAME = '.ci-cache-version'
-
-    @cached_nullary
-    def setup_dir(self) -> None:
-        version_file = os.path.join(self.dir, self.VERSION_FILE_NAME)
-
-        if self._config.no_create:
-            check.state(os.path.isdir(self.dir))
-
-        elif not os.path.isdir(self.dir):
-            os.makedirs(self.dir)
-            with open(version_file, 'w') as f:
-                f.write(str(self._version))
-            return
-
-        # NOTE: intentionally raises FileNotFoundError to refuse to use an existing non-cache dir as a cache dir.
-        with open(version_file) as f:
-            dir_version = int(f.read().strip())
-
-        if dir_version == self._version:
-            return
-
-        if self._config.no_purge:
-            raise RuntimeError(f'{dir_version=} != {self._version=}')
-
-        dirs = [n for n in sorted(os.listdir(self.dir)) if os.path.isdir(os.path.join(self.dir, n))]
-        if dirs:
-            raise RuntimeError(
-                f'Refusing to remove stale cache dir {self.dir!r} '
-                f'due to present directories: {", ".join(dirs)}',
-            )
+def build_docker_file_hash(docker_file: str) -> str:
+    with open(docker_file) as f:
+        contents = f.read()
 
-        for n in sorted(os.listdir(self.dir)):
-            if n.startswith('.'):
-                continue
-            fp = os.path.join(self.dir, n)
-            check.state(os.path.isfile(fp))
-            log.debug('Purging stale cache file: %s', fp)
-            os.unlink(fp)
+    return sha256_str(contents)
 
-        os.unlink(version_file)
 
-        with open(version_file, 'w') as f:
-            f.write(str(self._version))
+##
 
-    #
 
-    def get_cache_file_path(
-            self,
-            key: str,
-    ) -> str:
-        self.setup_dir()
-        return os.path.join(self.dir, key)
+def read_docker_tar_image_tag(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
+            m = mf.read()
 
-    def format_incomplete_file(self, f: str) -> str:
-        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+    manifests = json.loads(m.decode('utf-8'))
+    manifest = check.single(manifests)
+    tag = check.non_empty_str(check.single(manifest['RepoTags']))
+    return tag
 
-    #
 
-    async def get_file(self, key: str) -> ta.Optional[str]:
-        cache_file_path = self.get_cache_file_path(key)
-        if not os.path.exists(cache_file_path):
-            return None
-        return cache_file_path
+def read_docker_tar_image_id(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
+            i = mf.read()
 
-    async def put_file(
-            self,
-            key: str,
-            file_path: str,
-            *,
-            steal: bool = False,
-    ) -> str:
-        cache_file_path = self.get_cache_file_path(key)
-        if steal:
-            shutil.move(file_path, cache_file_path)
-        else:
-            shutil.copyfile(file_path, cache_file_path)
-        return cache_file_path
+    index = json.loads(i.decode('utf-8'))
+    manifest = check.single(index['manifests'])
+    image_id = check.non_empty_str(manifest['digest'])
+    return image_id
 
 
 ########################################
@@ -1560,99 +1526,33 @@ def is_in_github_actions() -> bool:
 
 
 ########################################
-# ../utils.py
+# ../../../omlish/argparse/cli.py
+"""
+TODO:
+ - default command
+ - auto match all underscores to hyphens
+ - pre-run, post-run hooks
+ - exitstack?
+"""
 
 
 ##
 
 
-def read_yaml_file(yaml_file: str) -> ta.Any:
-    yaml = __import__('yaml')
+@dc.dataclass(eq=False)
+class ArgparseArg:
+    args: ta.Sequence[ta.Any]
+    kwargs: ta.Mapping[str, ta.Any]
+    dest: ta.Optional[str] = None
 
-    with open(yaml_file) as f:
-        return yaml.safe_load(f)
+    def __get__(self, instance, owner=None):
+        if instance is None:
+            return self
+        return getattr(instance.args, self.dest)  # type: ignore
 
 
-##
-
-
-def sha256_str(s: str) -> str:
-    return hashlib.sha256(s.encode('utf-8')).hexdigest()
-
-
-##
-
-
-class LogTimingContext:
-    DEFAULT_LOG: ta.ClassVar[logging.Logger] = log
-
-    def __init__(
-            self,
-            description: str,
-            *,
-            log: ta.Optional[logging.Logger] = None,  # noqa
-            level: int = logging.DEBUG,
-    ) -> None:
-        super().__init__()
-
-        self._description = description
-        self._log = log if log is not None else self.DEFAULT_LOG
-        self._level = level
-
-    def set_description(self, description: str) -> 'LogTimingContext':
-        self._description = description
-        return self
-
-    _begin_time: float
-    _end_time: float
-
-    def __enter__(self) -> 'LogTimingContext':
-        self._begin_time = time.time()
-
-        self._log.log(self._level, f'Begin : {self._description}')  # noqa
-
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self._end_time = time.time()
-
-        self._log.log(
-            self._level,
-            f'End : {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
-        )
-
-
-log_timing_context = LogTimingContext
-
-
-########################################
-# ../../../omlish/argparse/cli.py
-"""
-TODO:
- - default command
- - auto match all underscores to hyphens
- - pre-run, post-run hooks
- - exitstack?
-"""
-
-
-##
-
-
-@dc.dataclass(eq=False)
-class ArgparseArg:
-    args: ta.Sequence[ta.Any]
-    kwargs: ta.Mapping[str, ta.Any]
-    dest: ta.Optional[str] = None
-
-    def __get__(self, instance, owner=None):
-        if instance is None:
-            return self
-        return getattr(instance.args, self.dest)  # type: ignore
-
-
-def argparse_arg(*args, **kwargs) -> ArgparseArg:
-    return ArgparseArg(args, kwargs)
+def argparse_arg(*args, **kwargs) -> ArgparseArg:
+    return ArgparseArg(args, kwargs)
 
 
 #
@@ -3115,6 +3015,15 @@ def check_lite_runtime_version() -> None:
         raise OSError(f'Requires python {LITE_REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}')  # noqa
 
 
+########################################
+# ../../../omlish/lite/timing.py
+
+
+LogTimingContext.DEFAULT_LOG = log
+
+log_timing_context = log_timing_context  # noqa
+
+
 ########################################
 # ../../../omlish/logs/json.py
 """
@@ -3215,47 +3124,263 @@ def temp_named_file_context(
 
 
 ########################################
-# ../docker/utils.py
-"""
-TODO:
- - some less stupid Dockerfile hash
-  - doesn't change too much though
-"""
+# ../cache.py
+
+
+CacheVersion = ta.NewType('CacheVersion', int)
 
 
 ##
 
 
-def build_docker_file_hash(docker_file: str) -> str:
-    with open(docker_file) as f:
-        contents = f.read()
+class FileCache(abc.ABC):
+    DEFAULT_CACHE_VERSION: ta.ClassVar[CacheVersion] = CacheVersion(CI_CACHE_VERSION)
 
-    return sha256_str(contents)
+    def __init__(
+            self,
+            *,
+            version: ta.Optional[CacheVersion] = None,
+    ) -> None:
+        super().__init__()
+
+        if version is None:
+            version = self.DEFAULT_CACHE_VERSION
+        check.isinstance(version, int)
+        check.arg(version >= 0)
+        self._version: CacheVersion = version
+
+    @property
+    def version(self) -> CacheVersion:
+        return self._version
+
+    #
+
+    @abc.abstractmethod
+    def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> ta.Awaitable[str]:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryFileCache(FileCache):
+    @dc.dataclass(frozen=True)
+    class Config:
+        dir: str
+
+        no_create: bool = False
+        no_purge: bool = False
+
+    def __init__(
+            self,
+            config: Config,
+            *,
+            version: ta.Optional[CacheVersion] = None,
+    ) -> None:  # noqa
+        super().__init__(
+            version=version,
+        )
+
+        self._config = config
+
+    @property
+    def dir(self) -> str:
+        return self._config.dir
+
+    #
+
+    VERSION_FILE_NAME = '.ci-cache-version'
+
+    @cached_nullary
+    def setup_dir(self) -> None:
+        version_file = os.path.join(self.dir, self.VERSION_FILE_NAME)
+
+        if self._config.no_create:
+            check.state(os.path.isdir(self.dir))
+
+        elif not os.path.isdir(self.dir):
+            os.makedirs(self.dir)
+            with open(version_file, 'w') as f:
+                f.write(str(self._version))
+            return
+
+        # NOTE: intentionally raises FileNotFoundError to refuse to use an existing non-cache dir as a cache dir.
+        with open(version_file) as f:
+            dir_version = int(f.read().strip())
+
+        if dir_version == self._version:
+            return
+
+        if self._config.no_purge:
+            raise RuntimeError(f'{dir_version=} != {self._version=}')
+
+        dirs = [n for n in sorted(os.listdir(self.dir)) if os.path.isdir(os.path.join(self.dir, n))]
+        if dirs:
+            raise RuntimeError(
+                f'Refusing to remove stale cache dir {self.dir!r} '
+                f'due to present directories: {", ".join(dirs)}',
+            )
+
+        for n in sorted(os.listdir(self.dir)):
+            if n.startswith('.'):
+                continue
+            fp = os.path.join(self.dir, n)
+            check.state(os.path.isfile(fp))
+            log.debug('Purging stale cache file: %s', fp)
+            os.unlink(fp)
+
+        os.unlink(version_file)
+
+        with open(version_file, 'w') as f:
+            f.write(str(self._version))
+
+    #
+
+    def get_cache_file_path(
+            self,
+            key: str,
+    ) -> str:
+        self.setup_dir()
+        return os.path.join(self.dir, key)
+
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+
+    #
+
+    async def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
+            return None
+        return cache_file_path
+
+    async def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = self.get_cache_file_path(key)
+        if steal:
+            shutil.move(file_path, cache_file_path)
+        else:
+            shutil.copyfile(file_path, cache_file_path)
+        return cache_file_path
 
 
 ##
 
 
-def read_docker_tar_image_tag(tar_file: str) -> str:
-    with tarfile.open(tar_file) as tf:
-        with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
-            m = mf.read()
+class DataCache:
+    @dc.dataclass(frozen=True)
+    class Data(abc.ABC):  # noqa
+        pass
 
-    manifests = json.loads(m.decode('utf-8'))
-    manifest = check.single(manifests)
-    tag = check.non_empty_str(check.single(manifest['RepoTags']))
-    return tag
+    @dc.dataclass(frozen=True)
+    class BytesData(Data):
+        data: bytes
 
+    @dc.dataclass(frozen=True)
+    class FileData(Data):
+        file_path: str
 
-def read_docker_tar_image_id(tar_file: str) -> str:
-    with tarfile.open(tar_file) as tf:
-        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
-            i = mf.read()
+    @dc.dataclass(frozen=True)
+    class UrlData(Data):
+        url: str
 
-    index = json.loads(i.decode('utf-8'))
-    manifest = check.single(index['manifests'])
-    image_id = check.non_empty_str(manifest['digest'])
-    return image_id
+    #
+
+    @abc.abstractmethod
+    def get_data(self, key: str) -> ta.Awaitable[ta.Optional[Data]]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def put_data(self, key: str, data: Data) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+
+#
+
+
+@functools.singledispatch
+async def read_data_cache_data(data: DataCache.Data) -> bytes:
+    raise TypeError(data)
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.BytesData) -> bytes:
+    return data.data
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.FileData) -> bytes:
+    with open(data.file_path, 'rb') as f:  # noqa
+        return f.read()
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.UrlData) -> bytes:
+    def inner() -> bytes:
+        with urllib.request.urlopen(urllib.request.Request(  # noqa
+            data.url,
+        )) as resp:
+            return resp.read()
+
+    return await asyncio.get_running_loop().run_in_executor(None, inner)
+
+
+#
+
+
+class FileCacheDataCache(DataCache):
+    def __init__(
+            self,
+            file_cache: FileCache,
+    ) -> None:
+        super().__init__()
+
+        self._file_cache = file_cache
+
+    async def get_data(self, key: str) -> ta.Optional[DataCache.Data]:
+        if (file_path := await self._file_cache.get_file(key)) is None:
+            return None
+
+        return DataCache.FileData(file_path)
+
+    async def put_data(self, key: str, data: DataCache.Data) -> None:
+        steal = False
+
+        if isinstance(data, DataCache.BytesData):
+            file_path = make_temp_file()
+            with open(file_path, 'wb') as f:  # noqa
+                f.write(data.data)
+            steal = True
+
+        elif isinstance(data, DataCache.FileData):
+            file_path = data.file_path
+
+        elif isinstance(data, DataCache.UrlData):
+            raise NotImplementedError
+
+        else:
+            raise TypeError(data)
+
+        await self._file_cache.put_file(
+            key,
+            file_path,
+            steal=steal,
+        )
 
 
 ########################################
@@ -3273,6 +3398,9 @@ class GithubCacheClient(abc.ABC):
     def get_entry(self, key: str) -> ta.Awaitable[ta.Optional[Entry]]:
         raise NotImplementedError
 
+    def get_entry_url(self, entry: Entry) -> ta.Optional[str]:
+        return None
+
     @abc.abstractmethod
     def download_file(self, entry: Entry, out_file: str) -> ta.Awaitable[None]:
         raise NotImplementedError
@@ -3339,7 +3467,7 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
     def _get_loop(self) -> asyncio.AbstractEventLoop:
         if (loop := self._given_loop) is not None:
             return loop
-        return asyncio.get_event_loop()
+        return asyncio.get_running_loop()
 
     #
 
@@ -3467,6 +3595,10 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
     class Entry(GithubCacheClient.Entry):
         artifact: GithubCacheServiceV1.ArtifactCacheEntry
 
+    def get_entry_url(self, entry: GithubCacheClient.Entry) -> ta.Optional[str]:
+        entry1 = check.isinstance(entry, self.Entry)
+        return entry1.artifact.cache_key
+
     #
 
     def build_get_entry_url_path(self, *keys: str) -> str:
@@ -4107,6 +4239,25 @@ class SubprocessRun:
     capture_output: ta.Optional[bool] = None
     kwargs: ta.Optional[ta.Mapping[str, ta.Any]] = None
 
+    @classmethod
+    def of(
+            cls,
+            *cmd: str,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+            check: bool = False,
+            capture_output: ta.Optional[bool] = None,
+            **kwargs: ta.Any,
+    ) -> 'SubprocessRun':
+        return cls(
+            cmd=cmd,
+            input=input,
+            timeout=timeout,
+            check=check,
+            capture_output=capture_output,
+            kwargs=kwargs,
+        )
+
 
 @dc.dataclass(frozen=True)
 class SubprocessRunOutput(ta.Generic[T]):
@@ -4340,7 +4491,7 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
 ##
 
 
-class GithubFileCache(FileCache):
+class GithubCache(FileCache, DataCache):
     @dc.dataclass(frozen=True)
     class Config:
         dir: str
@@ -4371,6 +4522,8 @@ class GithubFileCache(FileCache):
             version=self._version,
         )
 
+    #
+
     async def get_file(self, key: str) -> ta.Optional[str]:
         local_file = self._local.get_cache_file_path(key)
         if os.path.exists(local_file):
@@ -4404,6 +4557,21 @@ class GithubFileCache(FileCache):
 
         return cache_file_path
 
+    #
+
+    async def get_data(self, key: str) -> ta.Optional[DataCache.Data]:
+        local_file = self._local.get_cache_file_path(key)
+        if os.path.exists(local_file):
+            return DataCache.FileData(local_file)
+
+        if (entry := await self._client.get_entry(key)) is None:
+            return None
+
+        return DataCache.UrlData(check.non_empty_str(self._client.get_entry_url(entry)))
+
+    async def put_data(self, key: str, data: DataCache.Data) -> None:
+        await FileCacheDataCache(self).put_data(key, data)
+
 
 ########################################
 # ../github/cli.py
@@ -5019,11 +5187,11 @@ def bind_github(
 
     if cache_dir is not None:
         lst.extend([
-            inj.bind(GithubFileCache.Config(
+            inj.bind(GithubCache.Config(
                 dir=cache_dir,
             )),
-            inj.bind(GithubFileCache, singleton=True),
-            inj.bind(FileCache, to_key=GithubFileCache),
+            inj.bind(GithubCache, singleton=True),
+            inj.bind(FileCache, to_key=GithubCache),
         ])
 
     return inj.as_bindings(*lst)