ominfra 0.0.0.dev154__py3-none-any.whl → 0.0.0.dev156__py3-none-any.whl

ominfra/manage/deploy/venvs.py

@@ -0,0 +1,66 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - interp
+ - share more code with pyproject?
+"""
+import os.path
+import typing as ta
+
+from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
+
+from .paths import DeployPath
+from .paths import DeployPathOwner
+from .types import DeployAppTag
+from .types import DeployHome
+
+
+class DeployVenvManager(DeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: DeployHome,
+    ) -> None:
+        super().__init__()
+
+        self._deploy_home = deploy_home
+        self._dir = os.path.join(deploy_home, 'venvs')
+
+    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return {
+            DeployPath.parse('venvs/@app/@tag/'),
+        }
+
+    async def setup_venv(
+            self,
+            app_dir: str,
+            venv_dir: str,
+            *,
+            use_uv: bool = True,
+    ) -> None:
+        sys_exe = 'python3'
+
+        await asyncio_subprocesses.check_call(sys_exe, '-m', 'venv', venv_dir)
+
+        #
+
+        venv_exe = os.path.join(venv_dir, 'bin', 'python3')
+
+        #
+
+        reqs_txt = os.path.join(app_dir, 'requirements.txt')
+
+        if os.path.isfile(reqs_txt):
+            if use_uv:
+                await asyncio_subprocesses.check_call(venv_exe, '-m', 'pip', 'install', 'uv')
+                pip_cmd = ['-m', 'uv', 'pip']
+            else:
+                pip_cmd = ['-m', 'pip']
+
+            await asyncio_subprocesses.check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
+
+    async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
+        await self.setup_venv(
+            os.path.join(self._deploy_home, 'apps', app_tag.app, app_tag.tag),
+            os.path.join(self._deploy_home, 'venvs', app_tag.app, app_tag.tag),
+        )

ominfra/manage/inject.py

@@ -6,8 +6,10 @@ from omlish.lite.inject import InjectorBindings
 from omlish.lite.inject import inj
 from omlish.lite.marshal import ObjMarshalerManager
 
+from .bootstrap import MainBootstrap
 from .commands.inject import bind_commands
 from .config import MainConfig
+from .deploy.config import DeployConfig
 from .deploy.inject import bind_deploy
 from .marshal import ObjMarshalerInstaller
 from .marshal import ObjMarshalerInstallers
@@ -15,6 +17,7 @@ from .remote.config import RemoteConfig
 from .remote.inject import bind_remote
 from .system.config import SystemConfig
 from .system.inject import bind_system
+from .targets.inject import bind_targets
 
 
 ##
@@ -22,9 +25,13 @@ from .system.inject import bind_system
 
 def bind_main(
         *,
-        main_config: MainConfig,
-        remote_config: RemoteConfig,
-        system_config: SystemConfig,
+        main_config: MainConfig = MainConfig(),
+
+        deploy_config: DeployConfig = DeployConfig(),
+        remote_config: RemoteConfig = RemoteConfig(),
+        system_config: SystemConfig = SystemConfig(),
+
+        main_bootstrap: ta.Optional[MainBootstrap] = None,
 ) -> InjectorBindings:
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(main_config),
@@ -33,7 +40,9 @@ def bind_main(
             main_config=main_config,
         ),
 
-        bind_deploy(),
+        bind_deploy(
+            deploy_config=deploy_config,
+        ),
 
         bind_remote(
             remote_config=remote_config,
@@ -42,10 +51,17 @@ def bind_main(
         bind_system(
             system_config=system_config,
         ),
+
+        bind_targets(),
     ]
 
     #
 
+    if main_bootstrap is not None:
+        lst.append(inj.bind(main_bootstrap))
+
+    #
+
     def build_obj_marshaler_manager(insts: ObjMarshalerInstallers) -> ObjMarshalerManager:
         msh = ObjMarshalerManager()
         inst: ObjMarshalerInstaller

ominfra/manage/main.py

@@ -6,7 +6,6 @@ manage.py -s 'docker run -i python:3.12'
 manage.py -s 'ssh -i /foo/bar.pem foo@bar.baz' -q --python=python3.8
 """
 import asyncio
-import contextlib
 import json
 import sys
 import typing as ta
@@ -22,22 +21,17 @@ from omlish.lite.pycharm import PycharmRemoteDebug
 from .bootstrap import MainBootstrap
 from .bootstrap_ import main_bootstrap
 from .commands.base import Command
-from .commands.base import CommandExecutor
-from .commands.execution import LocalCommandExecutor
 from .config import MainConfig
+from .deploy.config import DeployConfig
 from .remote.config import RemoteConfig
-from .remote.connection import RemoteExecutionConnector
-from .remote.spawning import RemoteSpawning
+from .targets.connection import ManageTargetConnector
+from .targets.targets import ManageTarget
 
 
 class MainCli(ArgparseCli):
     @argparse_command(
         argparse_arg('--_payload-file'),
 
-        argparse_arg('-s', '--shell'),
-        argparse_arg('-q', '--shell-quote', action='store_true'),
-        argparse_arg('--python', default='python3'),
-
         argparse_arg('--pycharm-debug-port', type=int),
         argparse_arg('--pycharm-debug-host'),
         argparse_arg('--pycharm-debug-version'),
@@ -46,8 +40,9 @@ class MainCli(ArgparseCli):
 
         argparse_arg('--debug', action='store_true'),
 
-        argparse_arg('--local', action='store_true'),
+        argparse_arg('--deploy-home'),
 
+        argparse_arg('target'),
         argparse_arg('command', nargs='+'),
     )
     async def run(self) -> None:
@@ -58,6 +53,10 @@ class MainCli(ArgparseCli):
                 debug=bool(self.args.debug),
             ),
 
+            deploy_config=DeployConfig(
+                deploy_home=self.args.deploy_home,
+            ),
+
             remote_config=RemoteConfig(
                 payload_file=self.args._payload_file,  # noqa
 
@@ -68,8 +67,6 @@ class MainCli(ArgparseCli):
                 ) if self.args.pycharm_debug_port is not None else None,
 
                 timebomb_delay_s=self.args.remote_timebomb_delay_s,
-
-                use_in_process_remote_executor=True,
             ),
         )
 
@@ -83,6 +80,11 @@ class MainCli(ArgparseCli):
 
         msh = injector[ObjMarshalerManager]
 
+        ts = self.args.target
+        if not ts.startswith('{'):
+            ts = json.dumps({ts: {}})
+        tgt: ManageTarget = msh.unmarshal_obj(json.loads(ts), ManageTarget)
+
         cmds: ta.List[Command] = []
         cmd: Command
         for c in self.args.command:
@@ -93,21 +95,7 @@ class MainCli(ArgparseCli):
 
         #
 
-        async with contextlib.AsyncExitStack() as es:
-            ce: CommandExecutor
-
-            if self.args.local:
-                ce = injector[LocalCommandExecutor]
-
-            else:
-                tgt = RemoteSpawning.Target(
-                    shell=self.args.shell,
-                    shell_quote=self.args.shell_quote,
-                    python=self.args.python,
-                )
-
-                ce = await es.enter_async_context(injector[RemoteExecutionConnector].connect(tgt, bs))  # noqa
-
+        async with injector[ManageTargetConnector].connect(tgt) as ce:
             async def run_command(cmd: Command) -> None:
                 res = await ce.try_execute(
                     cmd,

ominfra/manage/remote/_main.py

@@ -20,7 +20,7 @@ from omlish.os.deathsig import set_process_deathsig
 
 from ...pyremote import pyremote_bootstrap_finalize
 from ..bootstrap import MainBootstrap
-from ..commands.execution import LocalCommandExecutor
+from ..commands.local import LocalCommandExecutor
 from .channel import RemoteChannel
 from .channel import RemoteChannelImpl
 from .execution import _RemoteCommandHandler

ominfra/manage/remote/config.py

@@ -20,5 +20,3 @@ class RemoteConfig:
     timebomb_delay_s: ta.Optional[float] = 60 * 60.
 
     heartbeat_interval_s: float = 3.
-
-    use_in_process_remote_executor: bool = False

ominfra/manage/remote/connection.py

@@ -1,18 +1,18 @@
 # ruff: noqa: UP006 UP007
-import abc
 import asyncio
 import contextlib
 import typing as ta
 
 from omlish.asyncs.asyncio.channels import asyncio_create_bytes_channel
 from omlish.lite.cached import cached_nullary
+from omlish.lite.contextmanagers import aclosing
 from omlish.lite.marshal import ObjMarshalerManager
 
 from ...pyremote import PyremoteBootstrapDriver
 from ...pyremote import PyremoteBootstrapOptions
 from ...pyremote import pyremote_build_bootstrap_cmd
 from ..bootstrap import MainBootstrap
-from ..commands.execution import LocalCommandExecutor
+from ..commands.local import LocalCommandExecutor
 from ._main import _remote_execution_main  # noqa
 from .channel import RemoteChannelImpl
 from .execution import RemoteCommandExecutor
@@ -25,20 +25,7 @@ from .spawning import RemoteSpawning
 ##
 
 
-class RemoteExecutionConnector(abc.ABC):
-    @abc.abstractmethod
-    def connect(
-            self,
-            tgt: RemoteSpawning.Target,
-            bs: MainBootstrap,
-    ) -> ta.AsyncContextManager[RemoteCommandExecutor]:
-        raise NotImplementedError
-
-
-##
-
-
-class PyremoteRemoteExecutionConnector(RemoteExecutionConnector):
+class PyremoteRemoteExecutionConnector:
     def __init__(
             self,
             *,
@@ -104,7 +91,7 @@ class PyremoteRemoteExecutionConnector(RemoteExecutionConnector):
             await chan.send_obj(bs)
 
             rce: RemoteCommandExecutor
-            async with contextlib.aclosing(RemoteCommandExecutor(chan)) as rce:
+            async with aclosing(RemoteCommandExecutor(chan)) as rce:
                 await rce.start()
 
                 yield rce
@@ -113,7 +100,7 @@ class PyremoteRemoteExecutionConnector(RemoteExecutionConnector):
 ##
 
 
-class InProcessRemoteExecutionConnector(RemoteExecutionConnector):
+class InProcessRemoteExecutionConnector:
     def __init__(
             self,
             *,
@@ -126,11 +113,7 @@ class InProcessRemoteExecutionConnector(RemoteExecutionConnector):
         self._local_executor = local_executor
 
     @contextlib.asynccontextmanager
-    async def connect(
-            self,
-            tgt: RemoteSpawning.Target,
-            bs: MainBootstrap,
-    ) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
+    async def connect(self) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
         r0, w0 = asyncio_create_bytes_channel()
         r1, w1 = asyncio_create_bytes_channel()
 
@@ -144,7 +127,7 @@ class InProcessRemoteExecutionConnector(RemoteExecutionConnector):
         rch_task = asyncio.create_task(rch.run())  # noqa
         try:
             rce: RemoteCommandExecutor
-            async with contextlib.aclosing(RemoteCommandExecutor(local_chan)) as rce:
+            async with aclosing(RemoteCommandExecutor(local_chan)) as rce:
                 await rce.start()
 
                 yield rce

ominfra/manage/remote/execution.py

@@ -394,7 +394,7 @@ class RemoteCommandExecutor(CommandExecutor):
             self,
             cmd: Command,
             *,
-            log: ta.Optional[logging.Logger] = None,
+            log: ta.Optional[logging.Logger] = None,  # noqa
             omit_exc_object: bool = False,
     ) -> CommandOutputOrException:
         try:

ominfra/manage/remote/inject.py

@@ -8,7 +8,6 @@ from omlish.lite.inject import inj
 from .config import RemoteConfig
 from .connection import InProcessRemoteExecutionConnector
 from .connection import PyremoteRemoteExecutionConnector
-from .connection import RemoteExecutionConnector
 from .payload import RemoteExecutionPayloadFile
 from .spawning import RemoteSpawning
 from .spawning import SubprocessRemoteSpawning
@@ -23,20 +22,10 @@ def bind_remote(
 
         inj.bind(SubprocessRemoteSpawning, singleton=True),
         inj.bind(RemoteSpawning, to_key=SubprocessRemoteSpawning),
-    ]
-
-    #
 
-    if remote_config.use_in_process_remote_executor:
-        lst.extend([
-            inj.bind(InProcessRemoteExecutionConnector, singleton=True),
-            inj.bind(RemoteExecutionConnector, to_key=InProcessRemoteExecutionConnector),
-        ])
-    else:
-        lst.extend([
-            inj.bind(PyremoteRemoteExecutionConnector, singleton=True),
-            inj.bind(RemoteExecutionConnector, to_key=PyremoteRemoteExecutionConnector),
-        ])
+        inj.bind(PyremoteRemoteExecutionConnector, singleton=True),
+        inj.bind(InProcessRemoteExecutionConnector, singleton=True),
+    ]
 
     #
 

ominfra/manage/remote/spawning.py

@@ -7,7 +7,7 @@ import shlex
 import subprocess
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_popen
+from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
 from omlish.lite.subprocesses import SubprocessChannelOption
@@ -86,7 +86,7 @@ class SubprocessRemoteSpawning(RemoteSpawning):
         if not debug:
             cmd = subprocess_maybe_shell_wrap_exec(*cmd)
 
-        async with asyncio_subprocess_popen(
+        async with asyncio_subprocesses.popen(
                 *cmd,
                 shell=pc.shell,
                 stdin=subprocess.PIPE,

ominfra/manage/system/commands.py

@@ -1,10 +1,14 @@
 # ruff: noqa: UP006 UP007
 import dataclasses as dc
+import typing as ta
 
+from omlish.lite.check import check
 from omlish.lite.logs import log
 
 from ..commands.base import Command
 from ..commands.base import CommandExecutor
+from .packages import SystemPackage
+from .packages import SystemPackageManager
 
 
 ##
@@ -12,13 +16,29 @@ from ..commands.base import CommandExecutor
 
 @dc.dataclass(frozen=True)
 class CheckSystemPackageCommand(Command['CheckSystemPackageCommand.Output']):
+    pkgs: ta.Sequence[str] = ()
+
+    def __post_init__(self) -> None:
+        check.not_isinstance(self.pkgs, str)
+
     @dc.dataclass(frozen=True)
     class Output(Command.Output):
-        pass
+        pkgs: ta.Sequence[SystemPackage]
 
 
 class CheckSystemPackageCommandExecutor(CommandExecutor[CheckSystemPackageCommand, CheckSystemPackageCommand.Output]):
+    def __init__(
+            self,
+            *,
+            mgr: SystemPackageManager,
+    ) -> None:
+        super().__init__()
+
+        self._mgr = mgr
+
     async def execute(self, cmd: CheckSystemPackageCommand) -> CheckSystemPackageCommand.Output:
         log.info('Checking system package!')
 
-        return CheckSystemPackageCommand.Output()
+        ret = await self._mgr.query(*cmd.pkgs)
+
+        return CheckSystemPackageCommand.Output(list(ret.values()))

ominfra/manage/system/config.py

@@ -2,7 +2,9 @@
 import dataclasses as dc
 import typing as ta
 
+from .platforms import Platform
+
 
 @dc.dataclass(frozen=True)
 class SystemConfig:
-    platform: ta.Optional[str] = None
+    platform: ta.Optional[Platform] = None

ominfra/manage/system/inject.py

@@ -1,5 +1,4 @@
 # ruff: noqa: UP006 UP007
-import sys
 import typing as ta
 
 from omlish.lite.inject import InjectorBindingOrBindings
@@ -13,7 +12,12 @@ from .config import SystemConfig
 from .packages import AptSystemPackageManager
 from .packages import BrewSystemPackageManager
 from .packages import SystemPackageManager
-from .types import SystemPlatform
+from .packages import YumSystemPackageManager
+from .platforms import AmazonLinuxPlatform
+from .platforms import DarwinPlatform
+from .platforms import LinuxPlatform
+from .platforms import Platform
+from .platforms import detect_system_platform
 
 
 def bind_system(
@@ -26,18 +30,24 @@ def bind_system(
 
     #
 
-    platform = system_config.platform or sys.platform
-    lst.append(inj.bind(platform, key=SystemPlatform))
+    platform = system_config.platform or detect_system_platform()
+    lst.append(inj.bind(platform, key=Platform))
 
     #
 
-    if platform == 'linux':
+    if isinstance(platform, AmazonLinuxPlatform):
+        lst.extend([
+            inj.bind(YumSystemPackageManager, singleton=True),
+            inj.bind(SystemPackageManager, to_key=YumSystemPackageManager),
+        ])
+
+    elif isinstance(platform, LinuxPlatform):
         lst.extend([
             inj.bind(AptSystemPackageManager, singleton=True),
             inj.bind(SystemPackageManager, to_key=AptSystemPackageManager),
         ])
 
-    elif platform == 'darwin':
+    elif isinstance(platform, DarwinPlatform):
         lst.extend([
             inj.bind(BrewSystemPackageManager, singleton=True),
             inj.bind(SystemPackageManager, to_key=BrewSystemPackageManager),

ominfra/manage/system/packages.py

@@ -9,9 +9,7 @@ import json
 import os
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_check_call
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_check_output
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_run
+from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 
 
@@ -44,10 +42,10 @@ class SystemPackageManager(abc.ABC):
 
 class BrewSystemPackageManager(SystemPackageManager):
     async def update(self) -> None:
-        await asyncio_subprocess_check_call('brew', 'update')
+        await asyncio_subprocesses.check_call('brew', 'update')
 
     async def upgrade(self) -> None:
-        await asyncio_subprocess_check_call('brew', 'upgrade')
+        await asyncio_subprocesses.check_call('brew', 'upgrade')
 
     async def install(self, *packages: SystemPackageOrStr) -> None:
         es: ta.List[str] = []
@@ -56,11 +54,11 @@ class BrewSystemPackageManager(SystemPackageManager):
                 es.append(p.name + (f'@{p.version}' if p.version is not None else ''))
             else:
                 es.append(p)
-        await asyncio_subprocess_check_call('brew', 'install', *es)
+        await asyncio_subprocesses.check_call('brew', 'install', *es)
 
     async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
         pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
-        o = await asyncio_subprocess_check_output('brew', 'info', '--json', *pns)
+        o = await asyncio_subprocesses.check_output('brew', 'info', '--json', *pns)
         j = json.loads(o.decode())
         d: ta.Dict[str, SystemPackage] = {}
         for e in j:
@@ -79,28 +77,54 @@ class AptSystemPackageManager(SystemPackageManager):
     }
 
     async def update(self) -> None:
-        await asyncio_subprocess_check_call('apt', 'update', env={**os.environ, **self._APT_ENV})
+        await asyncio_subprocesses.check_call('sudo', 'apt', 'update', env={**os.environ, **self._APT_ENV})
 
     async def upgrade(self) -> None:
-        await asyncio_subprocess_check_call('apt', 'upgrade', '-y', env={**os.environ, **self._APT_ENV})
+        await asyncio_subprocesses.check_call('sudo', 'apt', 'upgrade', '-y', env={**os.environ, **self._APT_ENV})
 
     async def install(self, *packages: SystemPackageOrStr) -> None:
         pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]  # FIXME: versions
-        await asyncio_subprocess_check_call('apt', 'install', '-y', *pns, env={**os.environ, **self._APT_ENV})
+        await asyncio_subprocesses.check_call('sudo', 'apt', 'install', '-y', *pns, env={**os.environ, **self._APT_ENV})
 
     async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
         pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
-        cmd = ['dpkg-query', '-W', '-f=${Package}=${Version}\n', *pns]
-        stdout, stderr = await asyncio_subprocess_run(
-            *cmd,
+        out = await asyncio_subprocesses.run(
+            'dpkg-query', '-W', '-f=${Package}=${Version}\n', *pns,
             capture_output=True,
             check=False,
         )
         d: ta.Dict[str, SystemPackage] = {}
-        for l in check.not_none(stdout).decode('utf-8').strip().splitlines():
+        for l in check.not_none(out.stdout).decode('utf-8').strip().splitlines():
             n, v = l.split('=', 1)
             d[n] = SystemPackage(
                 name=n,
                 version=v,
             )
         return d
+
+
+class YumSystemPackageManager(SystemPackageManager):
+    async def update(self) -> None:
+        await asyncio_subprocesses.check_call('sudo', 'yum', 'check-update')
+
+    async def upgrade(self) -> None:
+        await asyncio_subprocesses.check_call('sudo', 'yum', 'update')
+
+    async def install(self, *packages: SystemPackageOrStr) -> None:
+        pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]  # FIXME: versions
+        await asyncio_subprocesses.check_call('sudo', 'yum', 'install', *pns)
+
+    async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
+        pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
+        d: ta.Dict[str, SystemPackage] = {}
+        for pn in pns:
+            out = await asyncio_subprocesses.run(
+                'rpm', '-q', pn,
+                capture_output=True,
+            )
+            if not out.proc.returncode:
+                d[pn] = SystemPackage(
+                    pn,
+                    check.not_none(out.stdout).decode().strip(),
+                )
+        return d

ominfra/manage/system/platforms.py

@@ -0,0 +1,72 @@
+import abc
+import dataclasses as dc
+import sys
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.logs import log
+from omlish.os.linux import LinuxOsRelease
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Platform(abc.ABC):  # noqa
+    pass
+
+
+class LinuxPlatform(Platform, abc.ABC):
+    pass
+
+
+class UbuntuPlatform(LinuxPlatform):
+    pass
+
+
+class AmazonLinuxPlatform(LinuxPlatform):
+    pass
+
+
+class GenericLinuxPlatform(LinuxPlatform):
+    pass
+
+
+class DarwinPlatform(Platform):
+    pass
+
+
+class UnknownPlatform(Platform):
+    pass
+
+
+##
+
+
+def _detect_system_platform() -> Platform:
+    plat = sys.platform
+
+    if plat == 'linux':
+        if (osr := LinuxOsRelease.read()) is None:
+            return GenericLinuxPlatform()
+
+        if osr.id == 'amzn':
+            return AmazonLinuxPlatform()
+
+        elif osr.id == 'ubuntu':
+            return UbuntuPlatform()
+
+        else:
+            return GenericLinuxPlatform()
+
+    elif plat == 'darwin':
+        return DarwinPlatform()
+
+    else:
+        return UnknownPlatform()
+
+
+@cached_nullary
+def detect_system_platform() -> Platform:
+    platform = _detect_system_platform()
+    log.info('Detected platform: %r', platform)
+    return platform