ominfra 0.0.0.dev155__py3-none-any.whl → 0.0.0.dev156__py3-none-any.whl

ominfra/manage/commands/subprocess.py

@@ -6,8 +6,7 @@ import subprocess
 import time
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_communicate
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_popen
+from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
 from omlish.lite.subprocesses import SubprocessChannelOption
@@ -51,7 +50,7 @@ class SubprocessCommand(Command['SubprocessCommand.Output']):
 class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
     async def execute(self, cmd: SubprocessCommand) -> SubprocessCommand.Output:
         proc: asyncio.subprocess.Process
-        async with asyncio_subprocess_popen(
+        async with asyncio_subprocesses.popen(
             *subprocess_maybe_shell_wrap_exec(*cmd.cmd),
 
             shell=cmd.shell,
@@ -65,7 +64,7 @@ class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCom
             timeout=cmd.timeout,
         ) as proc:
             start_time = time.time()
-            stdout, stderr = await asyncio_subprocess_communicate(
+            stdout, stderr = await asyncio_subprocesses.communicate(
                 proc,
                 input=cmd.input,
                 timeout=cmd.timeout,

ominfra/manage/deploy/git.py

@@ -13,7 +13,7 @@ import functools
 import os.path
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_check_call
+from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.cached import async_cached_nullary
 from omlish.lite.check import check
 
@@ -92,7 +92,7 @@ class DeployGitManager(DeployPathOwner):
                 return f'https://{self._repo.host}/{self._repo.path}'
 
         async def _call(self, *cmd: str) -> None:
-            await asyncio_subprocess_check_call(
+            await asyncio_subprocesses.check_call(
                 *cmd,
                 cwd=self._dir,
             )
@@ -118,7 +118,7 @@ class DeployGitManager(DeployPathOwner):
             # FIXME: temp dir swap
             os.makedirs(dst_dir)
 
-            dst_call = functools.partial(asyncio_subprocess_check_call, cwd=dst_dir)
+            dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_dir)
             await dst_call('git', 'init')
 
             await dst_call('git', 'remote', 'add', 'local', self._dir)

ominfra/manage/deploy/venvs.py

@@ -7,7 +7,7 @@ TODO:
 import os.path
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_check_call
+from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
 
 from .paths import DeployPath
 from .paths import DeployPathOwner
@@ -40,7 +40,7 @@ class DeployVenvManager(DeployPathOwner):
     ) -> None:
         sys_exe = 'python3'
 
-        await asyncio_subprocess_check_call(sys_exe, '-m', 'venv', venv_dir)
+        await asyncio_subprocesses.check_call(sys_exe, '-m', 'venv', venv_dir)
 
         #
 
@@ -52,12 +52,12 @@ class DeployVenvManager(DeployPathOwner):
 
         if os.path.isfile(reqs_txt):
             if use_uv:
-                await asyncio_subprocess_check_call(venv_exe, '-m', 'pip', 'install', 'uv')
+                await asyncio_subprocesses.check_call(venv_exe, '-m', 'pip', 'install', 'uv')
                 pip_cmd = ['-m', 'uv', 'pip']
             else:
                 pip_cmd = ['-m', 'pip']
 
-            await asyncio_subprocess_check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
+            await asyncio_subprocesses.check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
 
     async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
         await self.setup_venv(

ominfra/manage/remote/spawning.py

@@ -7,7 +7,7 @@ import shlex
 import subprocess
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_popen
+from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
 from omlish.lite.subprocesses import SubprocessChannelOption
@@ -86,7 +86,7 @@ class SubprocessRemoteSpawning(RemoteSpawning):
         if not debug:
             cmd = subprocess_maybe_shell_wrap_exec(*cmd)
 
-        async with asyncio_subprocess_popen(
+        async with asyncio_subprocesses.popen(
                 *cmd,
                 shell=pc.shell,
                 stdin=subprocess.PIPE,

ominfra/manage/system/packages.py

@@ -9,9 +9,7 @@ import json
 import os
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_check_call
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_check_output
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_run
+from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 
 
@@ -44,10 +42,10 @@ class SystemPackageManager(abc.ABC):
 
 class BrewSystemPackageManager(SystemPackageManager):
     async def update(self) -> None:
-        await asyncio_subprocess_check_call('brew', 'update')
+        await asyncio_subprocesses.check_call('brew', 'update')
 
     async def upgrade(self) -> None:
-        await asyncio_subprocess_check_call('brew', 'upgrade')
+        await asyncio_subprocesses.check_call('brew', 'upgrade')
 
     async def install(self, *packages: SystemPackageOrStr) -> None:
         es: ta.List[str] = []
@@ -56,11 +54,11 @@ class BrewSystemPackageManager(SystemPackageManager):
                 es.append(p.name + (f'@{p.version}' if p.version is not None else ''))
             else:
                 es.append(p)
-        await asyncio_subprocess_check_call('brew', 'install', *es)
+        await asyncio_subprocesses.check_call('brew', 'install', *es)
 
     async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
         pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
-        o = await asyncio_subprocess_check_output('brew', 'info', '--json', *pns)
+        o = await asyncio_subprocesses.check_output('brew', 'info', '--json', *pns)
         j = json.loads(o.decode())
         d: ta.Dict[str, SystemPackage] = {}
         for e in j:
@@ -79,18 +77,18 @@ class AptSystemPackageManager(SystemPackageManager):
     }
 
     async def update(self) -> None:
-        await asyncio_subprocess_check_call('sudo', 'apt', 'update', env={**os.environ, **self._APT_ENV})
+        await asyncio_subprocesses.check_call('sudo', 'apt', 'update', env={**os.environ, **self._APT_ENV})
 
     async def upgrade(self) -> None:
-        await asyncio_subprocess_check_call('sudo', 'apt', 'upgrade', '-y', env={**os.environ, **self._APT_ENV})
+        await asyncio_subprocesses.check_call('sudo', 'apt', 'upgrade', '-y', env={**os.environ, **self._APT_ENV})
 
     async def install(self, *packages: SystemPackageOrStr) -> None:
         pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]  # FIXME: versions
-        await asyncio_subprocess_check_call('sudo', 'apt', 'install', '-y', *pns, env={**os.environ, **self._APT_ENV})
+        await asyncio_subprocesses.check_call('sudo', 'apt', 'install', '-y', *pns, env={**os.environ, **self._APT_ENV})
 
     async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
         pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
-        out = await asyncio_subprocess_run(
+        out = await asyncio_subprocesses.run(
             'dpkg-query', '-W', '-f=${Package}=${Version}\n', *pns,
             capture_output=True,
             check=False,
@@ -107,20 +105,20 @@ class AptSystemPackageManager(SystemPackageManager):
 
 class YumSystemPackageManager(SystemPackageManager):
     async def update(self) -> None:
-        await asyncio_subprocess_check_call('sudo', 'yum', 'check-update')
+        await asyncio_subprocesses.check_call('sudo', 'yum', 'check-update')
 
     async def upgrade(self) -> None:
-        await asyncio_subprocess_check_call('sudo', 'yum', 'update')
+        await asyncio_subprocesses.check_call('sudo', 'yum', 'update')
 
     async def install(self, *packages: SystemPackageOrStr) -> None:
         pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]  # FIXME: versions
-        await asyncio_subprocess_check_call('sudo', 'yum', 'install', *pns)
+        await asyncio_subprocesses.check_call('sudo', 'yum', 'install', *pns)
 
     async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
         pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
         d: ta.Dict[str, SystemPackage] = {}
         for pn in pns:
-            out = await asyncio_subprocess_run(
+            out = await asyncio_subprocesses.run(
                 'rpm', '-q', pn,
                 capture_output=True,
             )

ominfra/scripts/journald2aws.py

@@ -3593,168 +3593,222 @@ SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
 _SUBPROCESS_SHELL_WRAP_EXECS = False
 
 
-def subprocess_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
-    return ('sh', '-c', ' '.join(map(shlex.quote, args)))
+def subprocess_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
+    return ('sh', '-c', ' '.join(map(shlex.quote, cmd)))
 
 
-def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
+def subprocess_maybe_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
     if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
-        return subprocess_shell_wrap_exec(*args)
+        return subprocess_shell_wrap_exec(*cmd)
     else:
-        return args
-
-
-def prepare_subprocess_invocation(
-        *args: str,
-        env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
-        extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
-        quiet: bool = False,
-        shell: bool = False,
-        **kwargs: ta.Any,
-) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
-    log.debug('prepare_subprocess_invocation: args=%r', args)
-    if extra_env:
-        log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
-
-    if extra_env:
-        env = {**(env if env is not None else os.environ), **extra_env}
-
-    if quiet and 'stderr' not in kwargs:
-        if not log.isEnabledFor(logging.DEBUG):
-            kwargs['stderr'] = subprocess.DEVNULL
-
-    if not shell:
-        args = subprocess_maybe_shell_wrap_exec(*args)
-
-    return args, dict(
-        env=env,
-        shell=shell,
-        **kwargs,
-    )
+        return cmd
 
 
 ##
 
 
-@contextlib.contextmanager
-def subprocess_common_context(*args: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
-    start_time = time.time()
-    try:
-        log.debug('subprocess_common_context.try: args=%r', args)
-        yield
-
-    except Exception as exc:  # noqa
-        log.debug('subprocess_common_context.except: exc=%r', exc)
-        raise
+def subprocess_close(
+        proc: subprocess.Popen,
+        timeout: ta.Optional[float] = None,
+) -> None:
+    # TODO: terminate, sleep, kill
+    if proc.stdout:
+        proc.stdout.close()
+    if proc.stderr:
+        proc.stderr.close()
+    if proc.stdin:
+        proc.stdin.close()
 
-    finally:
-        end_time = time.time()
-        elapsed_s = end_time - start_time
-        log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
+    proc.wait(timeout)
 
 
 ##
 
 
-def subprocess_check_call(
-        *args: str,
-        stdout: ta.Any = sys.stderr,
-        **kwargs: ta.Any,
-) -> None:
-    args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
-    with subprocess_common_context(*args, **kwargs):
-        return subprocess.check_call(args, **kwargs)  # type: ignore
+class AbstractSubprocesses(abc.ABC):  # noqa
+    DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = log
+
+    def __init__(
+            self,
+            *,
+            log: ta.Optional[logging.Logger] = None,
+            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
+    ) -> None:
+        super().__init__()
 
+        self._log = log if log is not None else self.DEFAULT_LOGGER
+        self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
 
-def subprocess_check_output(
-        *args: str,
-        **kwargs: ta.Any,
-) -> bytes:
-    args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
-    with subprocess_common_context(*args, **kwargs):
-        return subprocess.check_output(args, **kwargs)
+    #
 
+    def prepare_args(
+            self,
+            *cmd: str,
+            env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
+            extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
+            quiet: bool = False,
+            shell: bool = False,
+            **kwargs: ta.Any,
+    ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
+        if self._log:
+            self._log.debug('Subprocesses.prepare_args: cmd=%r', cmd)
+            if extra_env:
+                self._log.debug('Subprocesses.prepare_args: extra_env=%r', extra_env)
 
-def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
-    return subprocess_check_output(*args, **kwargs).decode().strip()
+        if extra_env:
+            env = {**(env if env is not None else os.environ), **extra_env}
 
+        if quiet and 'stderr' not in kwargs:
+            if self._log and not self._log.isEnabledFor(logging.DEBUG):
+                kwargs['stderr'] = subprocess.DEVNULL
 
-##
+        if not shell:
+            cmd = subprocess_maybe_shell_wrap_exec(*cmd)
 
+        return cmd, dict(
+            env=env,
+            shell=shell,
+            **kwargs,
+        )
 
-DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
-    FileNotFoundError,
-    subprocess.CalledProcessError,
-)
+    @contextlib.contextmanager
+    def wrap_call(self, *cmd: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
+        start_time = time.time()
+        try:
+            if self._log:
+                self._log.debug('Subprocesses.wrap_call.try: cmd=%r', cmd)
+            yield
 
+        except Exception as exc:  # noqa
+            if self._log:
+                self._log.debug('Subprocesses.wrap_call.except: exc=%r', exc)
+            raise
 
-def _subprocess_try_run(
-        fn: ta.Callable[..., T],
-        *args: ta.Any,
-        try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
-        **kwargs: ta.Any,
-) -> ta.Union[T, Exception]:
-    try:
-        return fn(*args, **kwargs)
-    except try_exceptions as e:  # noqa
-        if log.isEnabledFor(logging.DEBUG):
-            log.exception('command failed')
-        return e
-
-
-def subprocess_try_call(
-        *args: str,
-        try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
-        **kwargs: ta.Any,
-) -> bool:
-    if isinstance(_subprocess_try_run(
-            subprocess_check_call,
-            *args,
-            try_exceptions=try_exceptions,
-            **kwargs,
-    ), Exception):
-        return False
-    else:
-        return True
+        finally:
+            end_time = time.time()
+            elapsed_s = end_time - start_time
+            if self._log:
+                self._log.debug('sSubprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
 
+    @contextlib.contextmanager
+    def prepare_and_wrap(
+            self,
+            *cmd: ta.Any,
+            **kwargs: ta.Any,
+    ) -> ta.Iterator[ta.Tuple[
+        ta.Tuple[ta.Any, ...],
+        ta.Dict[str, ta.Any],
+    ]]:
+        cmd, kwargs = self.prepare_args(*cmd, **kwargs)
+        with self.wrap_call(*cmd, **kwargs):
+            yield cmd, kwargs
 
-def subprocess_try_output(
-        *args: str,
-        try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
-        **kwargs: ta.Any,
-) -> ta.Optional[bytes]:
-    if isinstance(ret := _subprocess_try_run(
-            subprocess_check_output,
-            *args,
-            try_exceptions=try_exceptions,
-            **kwargs,
-    ), Exception):
-        return None
-    else:
-        return ret
+    #
+
+    DEFAULT_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
+        FileNotFoundError,
+        subprocess.CalledProcessError,
+    )
 
+    def try_fn(
+            self,
+            fn: ta.Callable[..., T],
+            *cmd: str,
+            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
+            **kwargs: ta.Any,
+    ) -> ta.Union[T, Exception]:
+        if try_exceptions is None:
+            try_exceptions = self._try_exceptions
 
-def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
-    out = subprocess_try_output(*args, **kwargs)
-    return out.decode().strip() if out is not None else None
+        try:
+            return fn(*cmd, **kwargs)
+
+        except try_exceptions as e:  # noqa
+            if self._log and self._log.isEnabledFor(logging.DEBUG):
+                self._log.exception('command failed')
+            return e
+
+    async def async_try_fn(
+            self,
+            fn: ta.Callable[..., ta.Awaitable[T]],
+            *cmd: ta.Any,
+            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
+            **kwargs: ta.Any,
+    ) -> ta.Union[T, Exception]:
+        if try_exceptions is None:
+            try_exceptions = self._try_exceptions
+
+        try:
+            return await fn(*cmd, **kwargs)
+
+        except try_exceptions as e:  # noqa
+            if self._log and self._log.isEnabledFor(logging.DEBUG):
+                self._log.exception('command failed')
+            return e
 
 
 ##
 
 
-def subprocess_close(
-        proc: subprocess.Popen,
-        timeout: ta.Optional[float] = None,
-) -> None:
-    # TODO: terminate, sleep, kill
-    if proc.stdout:
-        proc.stdout.close()
-    if proc.stderr:
-        proc.stderr.close()
-    if proc.stdin:
-        proc.stdin.close()
+class Subprocesses(AbstractSubprocesses):
+    def check_call(
+            self,
+            *cmd: str,
+            stdout: ta.Any = sys.stderr,
+            **kwargs: ta.Any,
+    ) -> None:
+        with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs):  # noqa
+            subprocess.check_call(cmd, **kwargs)
 
-    proc.wait(timeout)
+    def check_output(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> bytes:
+        with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs):  # noqa
+            return subprocess.check_output(cmd, **kwargs)
+
+    def check_output_str(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> str:
+        return self.check_output(*cmd, **kwargs).decode().strip()
+
+    #
+
+    def try_call(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> bool:
+        if isinstance(self.try_fn(self.check_call, *cmd, **kwargs), Exception):
+            return False
+        else:
+            return True
+
+    def try_output(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> ta.Optional[bytes]:
+        if isinstance(ret := self.try_fn(self.check_output, *cmd, **kwargs), Exception):
+            return None
+        else:
+            return ret
+
+    def try_output_str(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> ta.Optional[str]:
+        if (ret := self.try_output(*cmd, **kwargs)) is None:
+            return None
+        else:
+            return ret.decode().strip()
+
+
+subprocesses = Subprocesses()
 
 
 ########################################