omdev 0.0.0.dev223__py3-none-any.whl → 0.0.0.dev225__py3-none-any.whl

omdev/scripts/ci.py

@@ -89,7 +89,7 @@ InjectorProviderFn = ta.Callable[['Injector'], ta.Any]
 InjectorProviderFnMap = ta.Mapping['InjectorKey', 'InjectorProviderFn']
 InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
 
-# ../../omlish/subprocesses.py
+# ../../omlish/subprocesses/base.py
 SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']  # ta.TypeAlias
 
 
@@ -159,6 +159,27 @@ class ShellCmd:
         )
 
 
+########################################
+# ../utils.py
+
+
+##
+
+
+def read_yaml_file(yaml_file: str) -> ta.Any:
+    yaml = __import__('yaml')
+
+    with open(yaml_file) as f:
+        return yaml.safe_load(f)
+
+
+##
+
+
+def sha256_str(s: str) -> str:
+    return hashlib.sha256(s.encode('utf-8')).hexdigest()
+
+
 ########################################
 # ../../../omlish/asyncs/asyncio/asyncio.py
 
@@ -1146,6 +1167,63 @@ class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
         self._underlying.handleError(record)
 
 
+########################################
+# ../../../omlish/logs/timing.py
+
+
+##
+
+
+class LogTimingContext:
+    DEFAULT_LOG: ta.ClassVar[ta.Optional[logging.Logger]] = None
+
+    class _NOT_SPECIFIED:  # noqa
+        def __new__(cls, *args, **kwargs):  # noqa
+            raise TypeError
+
+    def __init__(
+            self,
+            description: str,
+            *,
+            log: ta.Union[logging.Logger, ta.Type[_NOT_SPECIFIED], None] = _NOT_SPECIFIED,  # noqa
+            level: int = logging.DEBUG,
+    ) -> None:
+        super().__init__()
+
+        self._description = description
+        if log is self._NOT_SPECIFIED:
+            log = self.DEFAULT_LOG  # noqa
+        self._log: ta.Optional[logging.Logger] = log  # type: ignore
+        self._level = level
+
+    def set_description(self, description: str) -> 'LogTimingContext':
+        self._description = description
+        return self
+
+    _begin_time: float
+    _end_time: float
+
+    def __enter__(self) -> 'LogTimingContext':
+        self._begin_time = time.time()
+
+        if self._log is not None:
+            self._log.log(self._level, f'Begin : {self._description}')  # noqa
+
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._end_time = time.time()
+
+        if self._log is not None:
+            self._log.log(
+                self._level,
+                f'End : {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
+            )
+
+
+log_timing_context = LogTimingContext
+
+
 ########################################
 # ../../../omlish/os/files.py
 
@@ -1183,159 +1261,47 @@ def unlinking_if_exists(path: str) -> ta.Iterator[None]:
 
 
 ########################################
-# ../cache.py
-
-
-CacheVersion = ta.NewType('CacheVersion', int)
+# ../docker/utils.py
+"""
+TODO:
+ - some less stupid Dockerfile hash
+  - doesn't change too much though
+"""
 
 
 ##
 
 
-class FileCache(abc.ABC):
-    DEFAULT_CACHE_VERSION: ta.ClassVar[CacheVersion] = CacheVersion(CI_CACHE_VERSION)
-
-    def __init__(
-            self,
-            *,
-            version: ta.Optional[CacheVersion] = None,
-    ) -> None:
-        super().__init__()
-
-        if version is None:
-            version = self.DEFAULT_CACHE_VERSION
-        check.isinstance(version, int)
-        check.arg(version >= 0)
-        self._version: CacheVersion = version
-
-    @property
-    def version(self) -> CacheVersion:
-        return self._version
-
-    #
-
-    @abc.abstractmethod
-    def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def put_file(
-            self,
-            key: str,
-            file_path: str,
-            *,
-            steal: bool = False,
-    ) -> ta.Awaitable[str]:
-        raise NotImplementedError
-
-
-#
-
-
-class DirectoryFileCache(FileCache):
-    @dc.dataclass(frozen=True)
-    class Config:
-        dir: str
-
-        no_create: bool = False
-        no_purge: bool = False
-
-    def __init__(
-            self,
-            config: Config,
-            *,
-            version: ta.Optional[CacheVersion] = None,
-    ) -> None:  # noqa
-        super().__init__(
-            version=version,
-        )
-
-        self._config = config
-
-    @property
-    def dir(self) -> str:
-        return self._config.dir
-
-    #
-
-    VERSION_FILE_NAME = '.ci-cache-version'
-
-    @cached_nullary
-    def setup_dir(self) -> None:
-        version_file = os.path.join(self.dir, self.VERSION_FILE_NAME)
-
-        if self._config.no_create:
-            check.state(os.path.isdir(self.dir))
-
-        elif not os.path.isdir(self.dir):
-            os.makedirs(self.dir)
-            with open(version_file, 'w') as f:
-                f.write(str(self._version))
-            return
-
-        # NOTE: intentionally raises FileNotFoundError to refuse to use an existing non-cache dir as a cache dir.
-        with open(version_file) as f:
-            dir_version = int(f.read().strip())
-
-        if dir_version == self._version:
-            return
-
-        if self._config.no_purge:
-            raise RuntimeError(f'{dir_version=} != {self._version=}')
-
-        dirs = [n for n in sorted(os.listdir(self.dir)) if os.path.isdir(os.path.join(self.dir, n))]
-        if dirs:
-            raise RuntimeError(
-                f'Refusing to remove stale cache dir {self.dir!r} '
-                f'due to present directories: {", ".join(dirs)}',
-            )
+def build_docker_file_hash(docker_file: str) -> str:
+    with open(docker_file) as f:
+        contents = f.read()
 
-        for n in sorted(os.listdir(self.dir)):
-            if n.startswith('.'):
-                continue
-            fp = os.path.join(self.dir, n)
-            check.state(os.path.isfile(fp))
-            log.debug('Purging stale cache file: %s', fp)
-            os.unlink(fp)
+    return sha256_str(contents)
 
-        os.unlink(version_file)
 
-        with open(version_file, 'w') as f:
-            f.write(str(self._version))
+##
 
-    #
 
-    def get_cache_file_path(
-            self,
-            key: str,
-    ) -> str:
-        self.setup_dir()
-        return os.path.join(self.dir, key)
+def read_docker_tar_image_tag(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
+            m = mf.read()
 
-    def format_incomplete_file(self, f: str) -> str:
-        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+    manifests = json.loads(m.decode('utf-8'))
+    manifest = check.single(manifests)
+    tag = check.non_empty_str(check.single(manifest['RepoTags']))
+    return tag
 
-    #
 
-    async def get_file(self, key: str) -> ta.Optional[str]:
-        cache_file_path = self.get_cache_file_path(key)
-        if not os.path.exists(cache_file_path):
-            return None
-        return cache_file_path
+def read_docker_tar_image_id(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
+            i = mf.read()
 
-    async def put_file(
-            self,
-            key: str,
-            file_path: str,
-            *,
-            steal: bool = False,
-    ) -> str:
-        cache_file_path = self.get_cache_file_path(key)
-        if steal:
-            shutil.move(file_path, cache_file_path)
-        else:
-            shutil.copyfile(file_path, cache_file_path)
-        return cache_file_path
+    index = json.loads(i.decode('utf-8'))
+    manifest = check.single(index['manifests'])
+    image_id = check.non_empty_str(manifest['digest'])
+    return image_id
 
 
 ########################################
@@ -1560,99 +1526,33 @@ def is_in_github_actions() -> bool:
 
 
 ########################################
-# ../utils.py
+# ../../../omlish/argparse/cli.py
+"""
+TODO:
+ - default command
+ - auto match all underscores to hyphens
+ - pre-run, post-run hooks
+ - exitstack?
+"""
 
 
 ##
 
 
-def read_yaml_file(yaml_file: str) -> ta.Any:
-    yaml = __import__('yaml')
+@dc.dataclass(eq=False)
+class ArgparseArg:
+    args: ta.Sequence[ta.Any]
+    kwargs: ta.Mapping[str, ta.Any]
+    dest: ta.Optional[str] = None
 
-    with open(yaml_file) as f:
-        return yaml.safe_load(f)
+    def __get__(self, instance, owner=None):
+        if instance is None:
+            return self
+        return getattr(instance.args, self.dest)  # type: ignore
 
 
-##
-
-
-def sha256_str(s: str) -> str:
-    return hashlib.sha256(s.encode('utf-8')).hexdigest()
-
-
-##
-
-
-class LogTimingContext:
-    DEFAULT_LOG: ta.ClassVar[logging.Logger] = log
-
-    def __init__(
-            self,
-            description: str,
-            *,
-            log: ta.Optional[logging.Logger] = None,  # noqa
-            level: int = logging.DEBUG,
-    ) -> None:
-        super().__init__()
-
-        self._description = description
-        self._log = log if log is not None else self.DEFAULT_LOG
-        self._level = level
-
-    def set_description(self, description: str) -> 'LogTimingContext':
-        self._description = description
-        return self
-
-    _begin_time: float
-    _end_time: float
-
-    def __enter__(self) -> 'LogTimingContext':
-        self._begin_time = time.time()
-
-        self._log.log(self._level, f'Begin : {self._description}')  # noqa
-
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self._end_time = time.time()
-
-        self._log.log(
-            self._level,
-            f'End : {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
-        )
-
-
-log_timing_context = LogTimingContext
-
-
-########################################
-# ../../../omlish/argparse/cli.py
-"""
-TODO:
- - default command
- - auto match all underscores to hyphens
- - pre-run, post-run hooks
- - exitstack?
-"""
-
-
-##
-
-
-@dc.dataclass(eq=False)
-class ArgparseArg:
-    args: ta.Sequence[ta.Any]
-    kwargs: ta.Mapping[str, ta.Any]
-    dest: ta.Optional[str] = None
-
-    def __get__(self, instance, owner=None):
-        if instance is None:
-            return self
-        return getattr(instance.args, self.dest)  # type: ignore
-
-
-def argparse_arg(*args, **kwargs) -> ArgparseArg:
-    return ArgparseArg(args, kwargs)
+def argparse_arg(*args, **kwargs) -> ArgparseArg:
+    return ArgparseArg(args, kwargs)
 
 
 #
@@ -3115,6 +3015,15 @@ def check_lite_runtime_version() -> None:
         raise OSError(f'Requires python {LITE_REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}')  # noqa
 
 
+########################################
+# ../../../omlish/lite/timing.py
+
+
+LogTimingContext.DEFAULT_LOG = log
+
+log_timing_context = log_timing_context  # noqa
+
+
 ########################################
 # ../../../omlish/logs/json.py
 """
@@ -3215,155 +3124,466 @@ def temp_named_file_context(
 
 
 ########################################
-# ../docker/utils.py
-"""
-TODO:
- - some less stupid Dockerfile hash
-  - doesn't change too much though
-"""
+# ../../../omlish/subprocesses/run.py
 
 
 ##
 
 
-def build_docker_file_hash(docker_file: str) -> str:
-    with open(docker_file) as f:
-        contents = f.read()
+@dc.dataclass(frozen=True)
+class SubprocessRunOutput(ta.Generic[T]):
+    proc: T
 
-    return sha256_str(contents)
+    returncode: int  # noqa
+
+    stdout: ta.Optional[bytes] = None
+    stderr: ta.Optional[bytes] = None
 
 
 ##
 
 
-def read_docker_tar_image_tag(tar_file: str) -> str:
-    with tarfile.open(tar_file) as tf:
-        with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
-            m = mf.read()
+@dc.dataclass(frozen=True)
+class SubprocessRun:
+    cmd: ta.Sequence[str]
+    input: ta.Any = None
+    timeout: ta.Optional[float] = None
+    check: bool = False
+    capture_output: ta.Optional[bool] = None
+    kwargs: ta.Optional[ta.Mapping[str, ta.Any]] = None
 
-    manifests = json.loads(m.decode('utf-8'))
-    manifest = check.single(manifests)
-    tag = check.non_empty_str(check.single(manifest['RepoTags']))
-    return tag
+    @classmethod
+    def of(
+            cls,
+            *cmd: str,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+            check: bool = False,  # noqa
+            capture_output: ta.Optional[bool] = None,
+            **kwargs: ta.Any,
+    ) -> 'SubprocessRun':
+        return cls(
+            cmd=cmd,
+            input=input,
+            timeout=timeout,
+            check=check,
+            capture_output=capture_output,
+            kwargs=kwargs,
+        )
 
+    #
 
-def read_docker_tar_image_id(tar_file: str) -> str:
-    with tarfile.open(tar_file) as tf:
-        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
-            i = mf.read()
+    _DEFAULT_SUBPROCESSES: ta.ClassVar[ta.Optional[ta.Any]] = None  # AbstractSubprocesses
 
-    index = json.loads(i.decode('utf-8'))
-    manifest = check.single(index['manifests'])
-    image_id = check.non_empty_str(manifest['digest'])
-    return image_id
+    def run(
+            self,
+            subprocesses: ta.Optional[ta.Any] = None,  # AbstractSubprocesses
+    ) -> SubprocessRunOutput:
+        if subprocesses is None:
+            subprocesses = self._DEFAULT_SUBPROCESSES
+        return check.not_none(subprocesses).run_(self)  # type: ignore[attr-defined]
 
+    _DEFAULT_ASYNC_SUBPROCESSES: ta.ClassVar[ta.Optional[ta.Any]] = None  # AbstractAsyncSubprocesses
 
-########################################
-# ../github/client.py
+    async def async_run(
+            self,
+            async_subprocesses: ta.Optional[ta.Any] = None,  # AbstractAsyncSubprocesses
+    ) -> SubprocessRunOutput:
+        if async_subprocesses is None:
+            async_subprocesses = self._DEFAULT_ASYNC_SUBPROCESSES
+        return await check.not_none(async_subprocesses).run_(self)  # type: ignore[attr-defined]
 
 
 ##
 
 
-class GithubCacheClient(abc.ABC):
-    class Entry(abc.ABC):  # noqa
-        pass
-
+class SubprocessRunnable(abc.ABC, ta.Generic[T]):
     @abc.abstractmethod
-    def get_entry(self, key: str) -> ta.Awaitable[ta.Optional[Entry]]:
+    def make_run(self) -> SubprocessRun:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def download_file(self, entry: Entry, out_file: str) -> ta.Awaitable[None]:
+    def handle_run_output(self, output: SubprocessRunOutput) -> T:
         raise NotImplementedError
 
-    @abc.abstractmethod
-    def upload_file(self, key: str, in_file: str) -> ta.Awaitable[None]:
-        raise NotImplementedError
+    #
 
+    def run(self, subprocesses: ta.Optional[ta.Any] = None) -> T:  # AbstractSubprocesses
+        return self.handle_run_output(self.make_run().run(subprocesses))
 
-##
+    async def async_run(self, async_subprocesses: ta.Optional[ta.Any] = None) -> T:  # AbstractAsyncSubprocesses
+        return self.handle_run_output(await self.make_run().async_run(async_subprocesses))
 
 
-class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
-    BASE_URL_ENV_VAR = register_github_env_var('ACTIONS_CACHE_URL')
-    AUTH_TOKEN_ENV_VAR = register_github_env_var('ACTIONS_RUNTIME_TOKEN')  # noqa
+########################################
+# ../cache.py
 
-    KEY_SUFFIX_ENV_VAR = register_github_env_var('GITHUB_RUN_ID')
 
-    #
+CacheVersion = ta.NewType('CacheVersion', int)
 
-    def __init__(
-            self,
-            *,
-            base_url: ta.Optional[str] = None,
-            auth_token: ta.Optional[str] = None,
 
-            key_prefix: ta.Optional[str] = None,
-            key_suffix: ta.Optional[str] = None,
+##
 
-            cache_version: int = CI_CACHE_VERSION,
 
-            loop: ta.Optional[asyncio.AbstractEventLoop] = None,
+class FileCache(abc.ABC):
+    DEFAULT_CACHE_VERSION: ta.ClassVar[CacheVersion] = CacheVersion(CI_CACHE_VERSION)
+
+    def __init__(
+            self,
+            *,
+            version: ta.Optional[CacheVersion] = None,
     ) -> None:
         super().__init__()
 
-        #
-
-        if base_url is None:
-            base_url = check.non_empty_str(self.BASE_URL_ENV_VAR())
-        self._service_url = GithubCacheServiceV1.get_service_url(base_url)
-
-        if auth_token is None:
-            auth_token = self.AUTH_TOKEN_ENV_VAR()
-        self._auth_token = auth_token
-
-        #
+        if version is None:
+            version = self.DEFAULT_CACHE_VERSION
+        check.isinstance(version, int)
+        check.arg(version >= 0)
+        self._version: CacheVersion = version
 
-        self._key_prefix = key_prefix
+    @property
+    def version(self) -> CacheVersion:
+        return self._version
 
-        if key_suffix is None:
-            key_suffix = self.KEY_SUFFIX_ENV_VAR()
-        self._key_suffix = check.non_empty_str(key_suffix)
+    #
 
-        #
+    @abc.abstractmethod
+    def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
+        raise NotImplementedError
 
-        self._cache_version = check.isinstance(cache_version, int)
+    @abc.abstractmethod
+    def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> ta.Awaitable[str]:
+        raise NotImplementedError
 
-        #
 
-        self._given_loop = loop
+#
 
-    #
 
-    def _get_loop(self) -> asyncio.AbstractEventLoop:
-        if (loop := self._given_loop) is not None:
-            return loop
-        return asyncio.get_event_loop()
+class DirectoryFileCache(FileCache):
+    @dc.dataclass(frozen=True)
+    class Config:
+        dir: str
 
-    #
+        no_create: bool = False
+        no_purge: bool = False
 
-    def build_request_headers(
+    def __init__(
             self,
-            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            config: Config,
             *,
-            content_type: ta.Optional[str] = None,
-            json_content: bool = False,
-    ) -> ta.Dict[str, str]:
-        dct = {
-            'Accept': ';'.join([
-                'application/json',
-                f'api-version={GithubCacheServiceV1.API_VERSION}',
-            ]),
-        }
+            version: ta.Optional[CacheVersion] = None,
+    ) -> None:  # noqa
+        super().__init__(
+            version=version,
+        )
 
-        if (auth_token := self._auth_token):
-            dct['Authorization'] = f'Bearer {auth_token}'
+        self._config = config
 
-        if content_type is None and json_content:
-            content_type = 'application/json'
-        if content_type is not None:
-            dct['Content-Type'] = content_type
+    @property
+    def dir(self) -> str:
+        return self._config.dir
+
+    #
+
+    VERSION_FILE_NAME = '.ci-cache-version'
+
+    @cached_nullary
+    def setup_dir(self) -> None:
+        version_file = os.path.join(self.dir, self.VERSION_FILE_NAME)
+
+        if self._config.no_create:
+            check.state(os.path.isdir(self.dir))
+
+        elif not os.path.isdir(self.dir):
+            os.makedirs(self.dir)
+            with open(version_file, 'w') as f:
+                f.write(str(self._version))
+            return
+
+        # NOTE: intentionally raises FileNotFoundError to refuse to use an existing non-cache dir as a cache dir.
+        with open(version_file) as f:
+            dir_version = int(f.read().strip())
+
+        if dir_version == self._version:
+            return
+
+        if self._config.no_purge:
+            raise RuntimeError(f'{dir_version=} != {self._version=}')
+
+        dirs = [n for n in sorted(os.listdir(self.dir)) if os.path.isdir(os.path.join(self.dir, n))]
+        if dirs:
+            raise RuntimeError(
+                f'Refusing to remove stale cache dir {self.dir!r} '
+                f'due to present directories: {", ".join(dirs)}',
+            )
+
+        for n in sorted(os.listdir(self.dir)):
+            if n.startswith('.'):
+                continue
+            fp = os.path.join(self.dir, n)
+            check.state(os.path.isfile(fp))
+            log.debug('Purging stale cache file: %s', fp)
+            os.unlink(fp)
+
+        os.unlink(version_file)
+
+        with open(version_file, 'w') as f:
+            f.write(str(self._version))
+
+    #
+
+    def get_cache_file_path(
+            self,
+            key: str,
+    ) -> str:
+        self.setup_dir()
+        return os.path.join(self.dir, key)
+
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+
+    #
+
+    async def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
+            return None
+        return cache_file_path
+
+    async def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = self.get_cache_file_path(key)
+        if steal:
+            shutil.move(file_path, cache_file_path)
+        else:
+            shutil.copyfile(file_path, cache_file_path)
+        return cache_file_path
+
+
+##
+
+
+class DataCache:
+    @dc.dataclass(frozen=True)
+    class Data(abc.ABC):  # noqa
+        pass
+
+    @dc.dataclass(frozen=True)
+    class BytesData(Data):
+        data: bytes
+
+    @dc.dataclass(frozen=True)
+    class FileData(Data):
+        file_path: str
+
+    @dc.dataclass(frozen=True)
+    class UrlData(Data):
+        url: str
+
+    #
+
+    @abc.abstractmethod
+    def get_data(self, key: str) -> ta.Awaitable[ta.Optional[Data]]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def put_data(self, key: str, data: Data) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+
+#
+
+
+@functools.singledispatch
+async def read_data_cache_data(data: DataCache.Data) -> bytes:
+    raise TypeError(data)
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.BytesData) -> bytes:
+    return data.data
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.FileData) -> bytes:
+    with open(data.file_path, 'rb') as f:  # noqa
+        return f.read()
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.UrlData) -> bytes:
+    def inner() -> bytes:
+        with urllib.request.urlopen(urllib.request.Request(  # noqa
+            data.url,
+        )) as resp:
+            return resp.read()
+
+    return await asyncio.get_running_loop().run_in_executor(None, inner)
+
+
+#
+
+
+class FileCacheDataCache(DataCache):
+    def __init__(
+            self,
+            file_cache: FileCache,
+    ) -> None:
+        super().__init__()
+
+        self._file_cache = file_cache
+
+    async def get_data(self, key: str) -> ta.Optional[DataCache.Data]:
+        if (file_path := await self._file_cache.get_file(key)) is None:
+            return None
+
+        return DataCache.FileData(file_path)
+
+    async def put_data(self, key: str, data: DataCache.Data) -> None:
+        steal = False
+
+        if isinstance(data, DataCache.BytesData):
+            file_path = make_temp_file()
+            with open(file_path, 'wb') as f:  # noqa
+                f.write(data.data)
+            steal = True
+
+        elif isinstance(data, DataCache.FileData):
+            file_path = data.file_path
+
+        elif isinstance(data, DataCache.UrlData):
+            raise NotImplementedError
+
+        else:
+            raise TypeError(data)
+
+        await self._file_cache.put_file(
+            key,
+            file_path,
+            steal=steal,
+        )
+
+
+########################################
+# ../github/client.py
+
+
+##
+
+
+class GithubCacheClient(abc.ABC):
+    class Entry(abc.ABC):  # noqa
+        pass
+
+    @abc.abstractmethod
+    def get_entry(self, key: str) -> ta.Awaitable[ta.Optional[Entry]]:
+        raise NotImplementedError
+
+    def get_entry_url(self, entry: Entry) -> ta.Optional[str]:
+        return None
+
+    @abc.abstractmethod
+    def download_file(self, entry: Entry, out_file: str) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def upload_file(self, key: str, in_file: str) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+
+##
+
+
+class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
+    BASE_URL_ENV_VAR = register_github_env_var('ACTIONS_CACHE_URL')
+    AUTH_TOKEN_ENV_VAR = register_github_env_var('ACTIONS_RUNTIME_TOKEN')  # noqa
+
+    KEY_SUFFIX_ENV_VAR = register_github_env_var('GITHUB_RUN_ID')
+
+    #
+
+    def __init__(
+            self,
+            *,
+            base_url: ta.Optional[str] = None,
+            auth_token: ta.Optional[str] = None,
+
+            key_prefix: ta.Optional[str] = None,
+            key_suffix: ta.Optional[str] = None,
+
+            cache_version: int = CI_CACHE_VERSION,
+
+            loop: ta.Optional[asyncio.AbstractEventLoop] = None,
+    ) -> None:
+        super().__init__()
+
+        #
+
+        if base_url is None:
+            base_url = check.non_empty_str(self.BASE_URL_ENV_VAR())
+        self._service_url = GithubCacheServiceV1.get_service_url(base_url)
+
+        if auth_token is None:
+            auth_token = self.AUTH_TOKEN_ENV_VAR()
+        self._auth_token = auth_token
+
+        #
+
+        self._key_prefix = key_prefix
+
+        if key_suffix is None:
+            key_suffix = self.KEY_SUFFIX_ENV_VAR()
+        self._key_suffix = check.non_empty_str(key_suffix)
+
+        #
+
+        self._cache_version = check.isinstance(cache_version, int)
+
+        #
+
+        self._given_loop = loop
+
+    #
+
+    def _get_loop(self) -> asyncio.AbstractEventLoop:
+        if (loop := self._given_loop) is not None:
+            return loop
+        return asyncio.get_running_loop()
+
+    #
+
+    def build_request_headers(
+            self,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            *,
+            content_type: ta.Optional[str] = None,
+            json_content: bool = False,
+    ) -> ta.Dict[str, str]:
+        dct = {
+            'Accept': ';'.join([
+                'application/json',
+                f'api-version={GithubCacheServiceV1.API_VERSION}',
+            ]),
+        }
+
+        if (auth_token := self._auth_token):
+            dct['Authorization'] = f'Bearer {auth_token}'
+
+        if content_type is None and json_content:
+            content_type = 'application/json'
+        if content_type is not None:
+            dct['Content-Type'] = content_type
 
         if headers:
             dct.update(headers)
@@ -3467,6 +3687,10 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
     class Entry(GithubCacheClient.Entry):
         artifact: GithubCacheServiceV1.ArtifactCacheEntry
 
+    def get_entry_url(self, entry: GithubCacheClient.Entry) -> ta.Optional[str]:
+        entry1 = check.isinstance(entry, self.Entry)
+        return entry1.artifact.cache_key
+
     #
 
     def build_get_entry_url_path(self, *keys: str) -> str:
@@ -3857,23 +4081,7 @@ def configure_standard_logging(
 
 
 ########################################
-# ../../../omlish/subprocesses.py
-
-
-##
-
-
-# Valid channel type kwarg values:
-#  - A special flag negative int
-#  - A positive fd int
-#  - A file-like object
-#  - None
-
-SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
-    'pipe': subprocess.PIPE,
-    'stdout': subprocess.STDOUT,
-    'devnull': subprocess.DEVNULL,
-}
+# ../../../omlish/subprocesses/wrap.py
 
 
 ##
@@ -3893,22 +4101,143 @@ def subprocess_maybe_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
         return cmd
 
 
+########################################
+# ../github/cache.py
+
+
 ##
 
 
-def subprocess_close(
-        proc: subprocess.Popen,
-        timeout: ta.Optional[float] = None,
-) -> None:
-    # TODO: terminate, sleep, kill
-    if proc.stdout:
-        proc.stdout.close()
-    if proc.stderr:
-        proc.stderr.close()
-    if proc.stdin:
-        proc.stdin.close()
+class GithubCache(FileCache, DataCache):
+    @dc.dataclass(frozen=True)
+    class Config:
+        dir: str
 
-    proc.wait(timeout)
+    def __init__(
+            self,
+            config: Config,
+            *,
+            client: ta.Optional[GithubCacheClient] = None,
+            version: ta.Optional[CacheVersion] = None,
+    ) -> None:
+        super().__init__(
+            version=version,
+        )
+
+        self._config = config
+
+        if client is None:
+            client = GithubCacheServiceV1Client(
+                cache_version=self._version,
+            )
+        self._client: GithubCacheClient = client
+
+        self._local = DirectoryFileCache(
+            DirectoryFileCache.Config(
+                dir=check.non_empty_str(config.dir),
+            ),
+            version=self._version,
+        )
+
+    #
+
+    async def get_file(self, key: str) -> ta.Optional[str]:
+        local_file = self._local.get_cache_file_path(key)
+        if os.path.exists(local_file):
+            return local_file
+
+        if (entry := await self._client.get_entry(key)) is None:
+            return None
+
+        tmp_file = self._local.format_incomplete_file(local_file)
+        with unlinking_if_exists(tmp_file):
+            await self._client.download_file(entry, tmp_file)
+
+            os.replace(tmp_file, local_file)
+
+        return local_file
+
+    async def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = await self._local.put_file(
+            key,
+            file_path,
+            steal=steal,
+        )
+
+        await self._client.upload_file(key, cache_file_path)
+
+        return cache_file_path
+
+    #
+
+    async def get_data(self, key: str) -> ta.Optional[DataCache.Data]:
+        local_file = self._local.get_cache_file_path(key)
+        if os.path.exists(local_file):
+            return DataCache.FileData(local_file)
+
+        if (entry := await self._client.get_entry(key)) is None:
+            return None
+
+        return DataCache.UrlData(check.non_empty_str(self._client.get_entry_url(entry)))
+
+    async def put_data(self, key: str, data: DataCache.Data) -> None:
+        await FileCacheDataCache(self).put_data(key, data)
+
+
+########################################
+# ../github/cli.py
+"""
+See:
+ - https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28
+"""
+
+
+class GithubCli(ArgparseCli):
+    @argparse_cmd()
+    def list_referenced_env_vars(self) -> None:
+        print('\n'.join(sorted(ev.k for ev in GITHUB_ENV_VARS)))
+
+    @argparse_cmd(
+        argparse_arg('key'),
+    )
+    async def get_cache_entry(self) -> None:
+        client = GithubCacheServiceV1Client()
+        entry = await client.get_entry(self.args.key)
+        if entry is None:
+            return
+        print(json_dumps_pretty(dc.asdict(entry)))  # noqa
+
+    @argparse_cmd(
+        argparse_arg('repository-id'),
+    )
+    def list_cache_entries(self) -> None:
+        raise NotImplementedError
+
+
+########################################
+# ../../../omlish/subprocesses/base.py
+
+
+##
+
+
+# Valid channel type kwarg values:
+#  - A special flag negative int
+#  - A positive fd int
+#  - A file-like object
+#  - None
+
+SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
+    'pipe': subprocess.PIPE,
+    'stdout': subprocess.STDOUT,
+    'devnull': subprocess.DEVNULL,
+}
 
 
 ##
@@ -4095,32 +4424,41 @@ class BaseSubprocesses(abc.ABC):  # noqa
             return e
 
 
+########################################
+# ../github/inject.py
+
+
 ##
 
 
-@dc.dataclass(frozen=True)
-class SubprocessRun:
-    cmd: ta.Sequence[str]
-    input: ta.Any = None
-    timeout: ta.Optional[float] = None
-    check: bool = False
-    capture_output: ta.Optional[bool] = None
-    kwargs: ta.Optional[ta.Mapping[str, ta.Any]] = None
+def bind_github(
+        *,
+        cache_dir: ta.Optional[str] = None,
+) -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = []
+
+    if cache_dir is not None:
+        lst.extend([
+            inj.bind(GithubCache.Config(
+                dir=cache_dir,
+            )),
+            inj.bind(GithubCache, singleton=True),
+            inj.bind(FileCache, to_key=GithubCache),
+        ])
 
+    return inj.as_bindings(*lst)
 
-@dc.dataclass(frozen=True)
-class SubprocessRunOutput(ta.Generic[T]):
-    proc: T
 
-    returncode: int  # noqa
+########################################
+# ../../../omlish/subprocesses/async_.py
 
-    stdout: ta.Optional[bytes] = None
-    stderr: ta.Optional[bytes] = None
 
+##
 
-class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
+
+class AbstractAsyncSubprocesses(BaseSubprocesses):
     @abc.abstractmethod
-    def run_(self, run: SubprocessRun) -> SubprocessRunOutput:
+    async def run_(self, run: SubprocessRun) -> SubprocessRunOutput:
         raise NotImplementedError
 
     def run(
@@ -4131,7 +4469,7 @@ class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
             check: bool = False,
             capture_output: ta.Optional[bool] = None,
             **kwargs: ta.Any,
-    ) -> SubprocessRunOutput:
+    ) -> ta.Awaitable[SubprocessRunOutput]:
         return self.run_(SubprocessRun(
             cmd=cmd,
             input=input,
@@ -4144,7 +4482,7 @@ class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
     #
 
     @abc.abstractmethod
-    def check_call(
+    async def check_call(
             self,
             *cmd: str,
             stdout: ta.Any = sys.stderr,
@@ -4153,7 +4491,7 @@ class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
         raise NotImplementedError
 
     @abc.abstractmethod
-    def check_output(
+    async def check_output(
             self,
             *cmd: str,
             **kwargs: ta.Any,
@@ -4162,96 +4500,56 @@ class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
 
     #
 
-    def check_output_str(
+    async def check_output_str(
             self,
             *cmd: str,
             **kwargs: ta.Any,
     ) -> str:
-        return self.check_output(*cmd, **kwargs).decode().strip()
+        return (await self.check_output(*cmd, **kwargs)).decode().strip()
 
     #
 
-    def try_call(
+    async def try_call(
             self,
             *cmd: str,
             **kwargs: ta.Any,
     ) -> bool:
-        if isinstance(self.try_fn(self.check_call, *cmd, **kwargs), Exception):
+        if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
             return False
         else:
             return True
 
-    def try_output(
+    async def try_output(
             self,
             *cmd: str,
             **kwargs: ta.Any,
     ) -> ta.Optional[bytes]:
-        if isinstance(ret := self.try_fn(self.check_output, *cmd, **kwargs), Exception):
+        if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
             return None
         else:
             return ret
 
-    def try_output_str(
+    async def try_output_str(
             self,
             *cmd: str,
             **kwargs: ta.Any,
     ) -> ta.Optional[str]:
-        if (ret := self.try_output(*cmd, **kwargs)) is None:
+        if (ret := await self.try_output(*cmd, **kwargs)) is None:
             return None
         else:
             return ret.decode().strip()
 
 
-##
-
-
-class Subprocesses(AbstractSubprocesses):
-    def run_(self, run: SubprocessRun) -> SubprocessRunOutput[subprocess.CompletedProcess]:
-        proc = subprocess.run(
-            run.cmd,
-            input=run.input,
-            timeout=run.timeout,
-            check=run.check,
-            capture_output=run.capture_output or False,
-            **(run.kwargs or {}),
-        )
-
-        return SubprocessRunOutput(
-            proc=proc,
-
-            returncode=proc.returncode,
-
-            stdout=proc.stdout,  # noqa
-            stderr=proc.stderr,  # noqa
-        )
-
-    def check_call(
-            self,
-            *cmd: str,
-            stdout: ta.Any = sys.stderr,
-            **kwargs: ta.Any,
-    ) -> None:
-        with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs):  # noqa
-            subprocess.check_call(cmd, **kwargs)
-
-    def check_output(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> bytes:
-        with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs):  # noqa
-            return subprocess.check_output(cmd, **kwargs)
-
-
-subprocesses = Subprocesses()
+########################################
+# ../../../omlish/subprocesses/sync.py
 
 
 ##
 
 
-class AbstractAsyncSubprocesses(BaseSubprocesses):
+class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
     @abc.abstractmethod
-    async def run_(self, run: SubprocessRun) -> SubprocessRunOutput:
+    def run_(self, run: SubprocessRun) -> SubprocessRunOutput:
         raise NotImplementedError
 
     def run(
@@ -4262,7 +4560,7 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
             check: bool = False,
             capture_output: ta.Optional[bool] = None,
             **kwargs: ta.Any,
-    ) -> ta.Awaitable[SubprocessRunOutput]:
+    ) -> SubprocessRunOutput:
         return self.run_(SubprocessRun(
             cmd=cmd,
             input=input,
@@ -4275,7 +4573,7 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
     #
 
     @abc.abstractmethod
-    async def check_call(
+    def check_call(
             self,
             *cmd: str,
             stdout: ta.Any = sys.stderr,
@@ -4284,7 +4582,7 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
         raise NotImplementedError
 
     @abc.abstractmethod
-    async def check_output(
+    def check_output(
             self,
             *cmd: str,
             **kwargs: ta.Any,
@@ -4293,146 +4591,94 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
 
     #
 
-    async def check_output_str(
+    def check_output_str(
             self,
             *cmd: str,
             **kwargs: ta.Any,
     ) -> str:
-        return (await self.check_output(*cmd, **kwargs)).decode().strip()
+        return self.check_output(*cmd, **kwargs).decode().strip()
 
     #
 
-    async def try_call(
+    def try_call(
             self,
             *cmd: str,
             **kwargs: ta.Any,
     ) -> bool:
-        if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
+        if isinstance(self.try_fn(self.check_call, *cmd, **kwargs), Exception):
             return False
         else:
             return True
 
-    async def try_output(
+    def try_output(
             self,
             *cmd: str,
             **kwargs: ta.Any,
     ) -> ta.Optional[bytes]:
-        if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
+        if isinstance(ret := self.try_fn(self.check_output, *cmd, **kwargs), Exception):
             return None
         else:
             return ret
 
-    async def try_output_str(
+    def try_output_str(
             self,
             *cmd: str,
             **kwargs: ta.Any,
     ) -> ta.Optional[str]:
-        if (ret := await self.try_output(*cmd, **kwargs)) is None:
+        if (ret := self.try_output(*cmd, **kwargs)) is None:
             return None
         else:
             return ret.decode().strip()
 
 
-########################################
-# ../github/cache.py
-
-
 ##
 
 
-class GithubFileCache(FileCache):
-    @dc.dataclass(frozen=True)
-    class Config:
-        dir: str
-
-    def __init__(
-            self,
-            config: Config,
-            *,
-            client: ta.Optional[GithubCacheClient] = None,
-            version: ta.Optional[CacheVersion] = None,
-    ) -> None:
-        super().__init__(
-            version=version,
-        )
+class Subprocesses(AbstractSubprocesses):
+    def run_(self, run: SubprocessRun) -> SubprocessRunOutput[subprocess.CompletedProcess]:
+        with self.prepare_and_wrap(
+                *run.cmd,
+                input=run.input,
+                timeout=run.timeout,
+                check=run.check,
+                capture_output=run.capture_output or False,
+                **(run.kwargs or {}),
+        ) as (cmd, kwargs):
+            proc = subprocess.run(cmd, **kwargs)  # noqa
 
-        self._config = config
+        return SubprocessRunOutput(
+            proc=proc,
 
-        if client is None:
-            client = GithubCacheServiceV1Client(
-                cache_version=self._version,
-            )
-        self._client: GithubCacheClient = client
+            returncode=proc.returncode,
 
-        self._local = DirectoryFileCache(
-            DirectoryFileCache.Config(
-                dir=check.non_empty_str(config.dir),
-            ),
-            version=self._version,
+            stdout=proc.stdout,  # noqa
+            stderr=proc.stderr,  # noqa
         )
 
-    async def get_file(self, key: str) -> ta.Optional[str]:
-        local_file = self._local.get_cache_file_path(key)
-        if os.path.exists(local_file):
-            return local_file
-
-        if (entry := await self._client.get_entry(key)) is None:
-            return None
-
-        tmp_file = self._local.format_incomplete_file(local_file)
-        with unlinking_if_exists(tmp_file):
-            await self._client.download_file(entry, tmp_file)
-
-            os.replace(tmp_file, local_file)
-
-        return local_file
-
-    async def put_file(
+    def check_call(
             self,
-            key: str,
-            file_path: str,
-            *,
-            steal: bool = False,
-    ) -> str:
-        cache_file_path = await self._local.put_file(
-            key,
-            file_path,
-            steal=steal,
-        )
-
-        await self._client.upload_file(key, cache_file_path)
-
-        return cache_file_path
+            *cmd: str,
+            stdout: ta.Any = sys.stderr,
+            **kwargs: ta.Any,
+    ) -> None:
+        with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs):  # noqa
+            subprocess.check_call(cmd, **kwargs)
 
+    def check_output(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> bytes:
+        with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs):  # noqa
+            return subprocess.check_output(cmd, **kwargs)
 
-########################################
-# ../github/cli.py
-"""
-See:
- - https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28
-"""
 
+##
 
-class GithubCli(ArgparseCli):
-    @argparse_cmd()
-    def list_referenced_env_vars(self) -> None:
-        print('\n'.join(sorted(ev.k for ev in GITHUB_ENV_VARS)))
 
-    @argparse_cmd(
-        argparse_arg('key'),
-    )
-    async def get_cache_entry(self) -> None:
-        client = GithubCacheServiceV1Client()
-        entry = await client.get_entry(self.args.key)
-        if entry is None:
-            return
-        print(json_dumps_pretty(dc.asdict(entry)))  # noqa
+subprocesses = Subprocesses()
 
-    @argparse_cmd(
-        argparse_arg('repository-id'),
-    )
-    def list_cache_entries(self) -> None:
-        raise NotImplementedError
+SubprocessRun._DEFAULT_SUBPROCESSES = subprocesses  # noqa
 
 
 ########################################
@@ -4648,19 +4894,19 @@ class AsyncioSubprocesses(AbstractAsyncSubprocesses):
             timeout: ta.Optional[float] = None,
             **kwargs: ta.Any,
     ) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
-        fac: ta.Any
-        if shell:
-            fac = functools.partial(
-                asyncio.create_subprocess_shell,
-                check.single(cmd),
-            )
-        else:
-            fac = functools.partial(
-                asyncio.create_subprocess_exec,
-                *cmd,
-            )
-
         with self.prepare_and_wrap( *cmd, shell=shell, **kwargs) as (cmd, kwargs):  # noqa
+            fac: ta.Any
+            if shell:
+                fac = functools.partial(
+                    asyncio.create_subprocess_shell,
+                    check.single(cmd),
+                )
+            else:
+                fac = functools.partial(
+                    asyncio.create_subprocess_exec,
+                    *cmd,
+                )
+
             proc: asyncio.subprocess.Process = await fac(**kwargs)
             try:
                 yield proc
@@ -5004,31 +5250,6 @@ async def load_docker_tar(
     return await load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))
 
 
-########################################
-# ../github/inject.py
-
-
-##
-
-
-def bind_github(
-        *,
-        cache_dir: ta.Optional[str] = None,
-) -> InjectorBindings:
-    lst: ta.List[InjectorBindingOrBindings] = []
-
-    if cache_dir is not None:
-        lst.extend([
-            inj.bind(GithubFileCache.Config(
-                dir=cache_dir,
-            )),
-            inj.bind(GithubFileCache, singleton=True),
-            inj.bind(FileCache, to_key=GithubFileCache),
-        ])
-
-    return inj.as_bindings(*lst)
-
-
 ########################################
 # ../docker/cache.py