ominfra 0.0.0.dev154__py3-none-any.whl → 0.0.0.dev155__py3-none-any.whl

ominfra/manage/bootstrap.py

@@ -1,6 +1,8 @@
+# ruff: noqa: UP006 UP007
 import dataclasses as dc
 
 from .config import MainConfig
+from .deploy.config import DeployConfig
 from .remote.config import RemoteConfig
 from .system.config import SystemConfig
 
@@ -9,6 +11,8 @@ from .system.config import SystemConfig
 class MainBootstrap:
     main_config: MainConfig = MainConfig()
 
+    deploy_config: DeployConfig = DeployConfig()
+
     remote_config: RemoteConfig = RemoteConfig()
 
     system_config: SystemConfig = SystemConfig()

ominfra/manage/bootstrap_.py

@@ -1,3 +1,4 @@
+# ruff: noqa: UP006 UP007
 from omlish.lite.inject import Injector
 from omlish.lite.inject import inj
 from omlish.lite.logs import configure_standard_logging
@@ -12,8 +13,12 @@ def main_bootstrap(bs: MainBootstrap) -> Injector:
 
     injector = inj.create_injector(bind_main(  # noqa
         main_config=bs.main_config,
+
+        deploy_config=bs.deploy_config,
         remote_config=bs.remote_config,
         system_config=bs.system_config,
+
+        main_bootstrap=bs,
     ))
 
     return injector

ominfra/manage/commands/inject.py

@@ -18,13 +18,13 @@ from .base import CommandNameMap
 from .base import CommandRegistration
 from .base import CommandRegistrations
 from .base import build_command_name_map
-from .execution import CommandExecutorMap
-from .execution import LocalCommandExecutor
-from .interp import InterpCommand
-from .interp import InterpCommandExecutor
+from .local import LocalCommandExecutor
 from .marshal import install_command_marshaling
+from .ping import PingCommand
+from .ping import PingCommandExecutor
 from .subprocess import SubprocessCommand
 from .subprocess import SubprocessCommandExecutor
+from .types import CommandExecutorMap
 
 
 ##
@@ -113,13 +113,10 @@ def bind_commands(
 
     #
 
-    command_cls: ta.Any
-    executor_cls: ta.Any
-    for command_cls, executor_cls in [
-        (SubprocessCommand, SubprocessCommandExecutor),
-        (InterpCommand, InterpCommandExecutor),
-    ]:
-        lst.append(bind_command(command_cls, executor_cls))
+    lst.extend([
+        bind_command(PingCommand, PingCommandExecutor),
+        bind_command(SubprocessCommand, SubprocessCommandExecutor),
+    ])
 
     #
 

ominfra/manage/commands/{execution.py → local.py}

@@ -1,11 +1,7 @@
 # ruff: noqa: UP006 UP007
-import typing as ta
-
 from .base import Command
 from .base import CommandExecutor
-
-
-CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
+from .types import CommandExecutorMap
 
 
 class LocalCommandExecutor(CommandExecutor):

ominfra/manage/commands/ping.py

@@ -0,0 +1,23 @@
+# ruff: noqa: TC003 UP006 UP007
+import dataclasses as dc
+import time
+
+from .base import Command
+from .base import CommandExecutor
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class PingCommand(Command['PingCommand.Output']):
+    time: float = dc.field(default_factory=time.time)
+
+    @dc.dataclass(frozen=True)
+    class Output(Command.Output):
+        time: float
+
+
+class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
+    async def execute(self, cmd: PingCommand) -> PingCommand.Output:
+        return PingCommand.Output(cmd.time)

ominfra/manage/commands/types.py

@@ -0,0 +1,8 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from .base import Command
+from .base import CommandExecutor
+
+
+CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])

ominfra/manage/deploy/apps.py

@@ -0,0 +1,72 @@
+# ruff: noqa: UP006 UP007
+import datetime
+import os.path
+import typing as ta
+
+from .git import DeployGitManager
+from .git import DeployGitRepo
+from .git import DeployGitSpec
+from .paths import DeployPath
+from .paths import DeployPathOwner
+from .types import DeployApp
+from .types import DeployAppTag
+from .types import DeployHome
+from .types import DeployRev
+from .types import DeployTag
+from .venvs import DeployVenvManager
+
+
+def make_deploy_tag(
+        rev: DeployRev,
+        now: ta.Optional[datetime.datetime] = None,
+) -> DeployTag:
+    if now is None:
+        now = datetime.datetime.utcnow()  # noqa
+    now_fmt = '%Y%m%dT%H%M%S'
+    now_str = now.strftime(now_fmt)
+    return DeployTag('-'.join([rev, now_str]))
+
+
+class DeployAppManager(DeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: DeployHome,
+            git: DeployGitManager,
+            venvs: DeployVenvManager,
+    ) -> None:
+        super().__init__()
+
+        self._deploy_home = deploy_home
+        self._git = git
+        self._venvs = venvs
+
+        self._dir = os.path.join(deploy_home, 'apps')
+
+    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return {
+            DeployPath.parse('apps/@app/@tag'),
+        }
+
+    async def prepare_app(
+            self,
+            app: DeployApp,
+            rev: DeployRev,
+            repo: DeployGitRepo,
+    ):
+        app_tag = DeployAppTag(app, make_deploy_tag(rev))
+        app_dir = os.path.join(self._dir, app, app_tag.tag)
+
+        #
+
+        await self._git.checkout(
+            DeployGitSpec(
+                repo=repo,
+                rev=rev,
+            ),
+            app_dir,
+        )
+
+        #
+
+        await self._venvs.setup_app_venv(app_tag)

ominfra/manage/deploy/config.py

@@ -0,0 +1,8 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import typing as ta
+
+
+@dc.dataclass(frozen=True)
+class DeployConfig:
+    deploy_home: ta.Optional[str] = None

ominfra/manage/deploy/git.py

@@ -0,0 +1,136 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - 'repos'?
+
+git/github.com/wrmsr/omlish <- bootstrap repo
+ - shallow clone off bootstrap into /apps
+
+github.com/wrmsr/omlish@rev
+"""
+import dataclasses as dc
+import functools
+import os.path
+import typing as ta
+
+from omlish.lite.asyncio.subprocesses import asyncio_subprocess_check_call
+from omlish.lite.cached import async_cached_nullary
+from omlish.lite.check import check
+
+from .paths import DeployPath
+from .paths import DeployPathOwner
+from .types import DeployHome
+from .types import DeployRev
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployGitRepo:
+    host: ta.Optional[str] = None
+    username: ta.Optional[str] = None
+    path: ta.Optional[str] = None
+
+    def __post_init__(self) -> None:
+        check.not_in('..', check.non_empty_str(self.host))
+        check.not_in('.', check.non_empty_str(self.path))
+
+
+@dc.dataclass(frozen=True)
+class DeployGitSpec:
+    repo: DeployGitRepo
+    rev: DeployRev
+
+
+##
+
+
+class DeployGitManager(DeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: DeployHome,
+    ) -> None:
+        super().__init__()
+
+        self._deploy_home = deploy_home
+        self._dir = os.path.join(deploy_home, 'git')
+
+        self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
+
+    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return {
+            DeployPath.parse('git'),
+        }
+
+    class RepoDir:
+        def __init__(
+                self,
+                git: 'DeployGitManager',
+                repo: DeployGitRepo,
+        ) -> None:
+            super().__init__()
+
+            self._git = git
+            self._repo = repo
+            self._dir = os.path.join(
+                self._git._dir,  # noqa
+                check.non_empty_str(repo.host),
+                check.non_empty_str(repo.path),
+            )
+
+        @property
+        def repo(self) -> DeployGitRepo:
+            return self._repo
+
+        @property
+        def url(self) -> str:
+            if self._repo.username is not None:
+                return f'{self._repo.username}@{self._repo.host}:{self._repo.path}'
+            else:
+                return f'https://{self._repo.host}/{self._repo.path}'
+
+        async def _call(self, *cmd: str) -> None:
+            await asyncio_subprocess_check_call(
+                *cmd,
+                cwd=self._dir,
+            )
+
+        @async_cached_nullary
+        async def init(self) -> None:
+            os.makedirs(self._dir, exist_ok=True)
+            if os.path.exists(os.path.join(self._dir, '.git')):
+                return
+
+            await self._call('git', 'init')
+            await self._call('git', 'remote', 'add', 'origin', self.url)
+
+        async def fetch(self, rev: DeployRev) -> None:
+            await self.init()
+            await self._call('git', 'fetch', '--depth=1', 'origin', rev)
+
+        async def checkout(self, rev: DeployRev, dst_dir: str) -> None:
+            check.state(not os.path.exists(dst_dir))
+
+            await self.fetch(rev)
+
+            # FIXME: temp dir swap
+            os.makedirs(dst_dir)
+
+            dst_call = functools.partial(asyncio_subprocess_check_call, cwd=dst_dir)
+            await dst_call('git', 'init')
+
+            await dst_call('git', 'remote', 'add', 'local', self._dir)
+            await dst_call('git', 'fetch', '--depth=1', 'local', rev)
+            await dst_call('git', 'checkout', rev)
+
+    def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
+        try:
+            return self._repo_dirs[repo]
+        except KeyError:
+            repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
+            return repo_dir
+
+    async def checkout(self, spec: DeployGitSpec, dst_dir: str) -> None:
+        await self.get_repo_dir(spec.repo).checkout(spec.rev, dst_dir)

ominfra/manage/deploy/inject.py

@@ -1,4 +1,5 @@
 # ruff: noqa: UP006 UP007
+import os.path
 import typing as ta
 
 from omlish.lite.inject import InjectorBindingOrBindings
@@ -6,14 +7,34 @@ from omlish.lite.inject import InjectorBindings
 from omlish.lite.inject import inj
 
 from ..commands.inject import bind_command
+from .apps import DeployAppManager
 from .commands import DeployCommand
 from .commands import DeployCommandExecutor
+from .config import DeployConfig
+from .git import DeployGitManager
+from .interp import InterpCommand
+from .interp import InterpCommandExecutor
+from .types import DeployHome
+from .venvs import DeployVenvManager
 
 
 def bind_deploy(
+        *,
+        deploy_config: DeployConfig,
 ) -> InjectorBindings:
     lst: ta.List[InjectorBindingOrBindings] = [
+        inj.bind(deploy_config),
+
+        inj.bind(DeployAppManager, singleton=True),
+        inj.bind(DeployGitManager, singleton=True),
+        inj.bind(DeployVenvManager, singleton=True),
+
         bind_command(DeployCommand, DeployCommandExecutor),
+        bind_command(InterpCommand, InterpCommandExecutor),
     ]
 
+    if (dh := deploy_config.deploy_home) is not None:
+        dh = os.path.abspath(os.path.expanduser(dh))
+        lst.append(inj.bind(dh, key=DeployHome))
+
     return inj.as_bindings(*lst)

ominfra/manage/deploy/paths.py

@@ -30,6 +30,9 @@ for dn in [
     'conf/supervisor',
     'venv',
 ]:
+
+==
+
 """
 import abc
 import dataclasses as dc
@@ -40,12 +43,23 @@ from omlish.lite.check import check
 
 
 DeployPathKind = ta.Literal['dir', 'file']  # ta.TypeAlias
+DeployPathSpec = ta.Literal['app', 'tag']  # ta.TypeAlias
 
 
 ##
 
 
 DEPLOY_PATH_SPEC_PLACEHOLDER = '@'
+DEPLOY_PATH_SPEC_SEPARATORS = '-.'
+
+DEPLOY_PATH_SPECS: ta.FrozenSet[str] = frozenset([
+    'app',
+    'tag',  # <rev>-<dt>
+])
+
+
+class DeployPathError(Exception):
+    pass
 
 
 @dc.dataclass(frozen=True)
@@ -56,39 +70,43 @@ class DeployPathPart(abc.ABC):  # noqa
         raise NotImplementedError
 
     @abc.abstractmethod
-    def render(self) -> str:
+    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
         raise NotImplementedError
 
 
 #
 
 
-class DeployPathDir(DeployPathPart, abc.ABC):
+class DirDeployPathPart(DeployPathPart, abc.ABC):
     @property
     def kind(self) -> DeployPathKind:
         return 'dir'
 
     @classmethod
-    def parse(cls, s: str) -> 'DeployPathDir':
+    def parse(cls, s: str) -> 'DirDeployPathPart':
         if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
-            check.equal(s, DEPLOY_PATH_SPEC_PLACEHOLDER)
-            return SpecDeployPathDir()
+            check.equal(s[0], DEPLOY_PATH_SPEC_PLACEHOLDER)
+            return SpecDirDeployPathPart(s[1:])
         else:
-            return ConstDeployPathDir(s)
+            return ConstDirDeployPathPart(s)
 
 
-class DeployPathFile(DeployPathPart, abc.ABC):
+class FileDeployPathPart(DeployPathPart, abc.ABC):
     @property
     def kind(self) -> DeployPathKind:
         return 'file'
 
     @classmethod
-    def parse(cls, s: str) -> 'DeployPathFile':
+    def parse(cls, s: str) -> 'FileDeployPathPart':
         if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
             check.equal(s[0], DEPLOY_PATH_SPEC_PLACEHOLDER)
-            return SpecDeployPathFile(s[1:])
+            if not any(c in s for c in DEPLOY_PATH_SPEC_SEPARATORS):
+                return SpecFileDeployPathPart(s[1:], '')
+            else:
+                p = min(f for c in DEPLOY_PATH_SPEC_SEPARATORS if (f := s.find(c)) > 0)
+                return SpecFileDeployPathPart(s[1:p], s[p:])
         else:
-            return ConstDeployPathFile(s)
+            return ConstFileDeployPathPart(s)
 
 
 #
@@ -103,41 +121,56 @@ class ConstDeployPathPart(DeployPathPart, abc.ABC):
         check.not_in('/', self.name)
         check.not_in(DEPLOY_PATH_SPEC_PLACEHOLDER, self.name)
 
-    def render(self) -> str:
+    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
         return self.name
 
 
-class ConstDeployPathDir(ConstDeployPathPart, DeployPathDir):
+class ConstDirDeployPathPart(ConstDeployPathPart, DirDeployPathPart):
     pass
 
 
-class ConstDeployPathFile(ConstDeployPathPart, DeployPathFile):
+class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
     pass
 
 
 #
 
 
+@dc.dataclass(frozen=True)
 class SpecDeployPathPart(DeployPathPart, abc.ABC):
-    pass
+    spec: str  # DeployPathSpec
+
+    def __post_init__(self) -> None:
+        check.non_empty_str(self.spec)
+        for c in [*DEPLOY_PATH_SPEC_SEPARATORS, DEPLOY_PATH_SPEC_PLACEHOLDER, '/']:
+            check.not_in(c, self.spec)
+        check.in_(self.spec, DEPLOY_PATH_SPECS)
+
+    def _render_spec(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+        if specs is not None:
+            return specs[self.spec]  # type: ignore
+        else:
+            return DEPLOY_PATH_SPEC_PLACEHOLDER + self.spec
 
 
-class SpecDeployPathDir(SpecDeployPathPart, DeployPathDir):
-    def render(self) -> str:
-        return DEPLOY_PATH_SPEC_PLACEHOLDER
+@dc.dataclass(frozen=True)
+class SpecDirDeployPathPart(SpecDeployPathPart, DirDeployPathPart):
+    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+        return self._render_spec(specs)
 
 
 @dc.dataclass(frozen=True)
-class SpecDeployPathFile(SpecDeployPathPart, DeployPathFile):
+class SpecFileDeployPathPart(SpecDeployPathPart, FileDeployPathPart):
     suffix: str
 
     def __post_init__(self) -> None:
-        check.non_empty_str(self.suffix)
-        check.not_in('/', self.suffix)
-        check.not_in(DEPLOY_PATH_SPEC_PLACEHOLDER, self.suffix)
+        super().__post_init__()
+        if self.suffix:
+            for c in [DEPLOY_PATH_SPEC_PLACEHOLDER, '/']:
+                check.not_in(c, self.suffix)
 
-    def render(self) -> str:
-        return DEPLOY_PATH_SPEC_PLACEHOLDER + self.suffix
+    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+        return self._render_spec(specs) + self.suffix
 
 
 ##
@@ -152,13 +185,24 @@ class DeployPath:
         for p in self.parts[:-1]:
             check.equal(p.kind, 'dir')
 
+        pd = {}
+        for i, p in enumerate(self.parts):
+            if isinstance(p, SpecDeployPathPart):
+                if p.spec in pd:
+                    raise DeployPathError('Duplicate specs in path', self)
+                pd[p.spec] = i
+
+        if 'tag' in pd:
+            if 'app' not in pd or pd['app'] >= pd['tag']:
+                raise DeployPathError('Tag spec in path without preceding app', self)
+
     @property
     def kind(self) -> ta.Literal['file', 'dir']:
         return self.parts[-1].kind
 
-    def render(self) -> str:
+    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
         return os.path.join(  # noqa
-            *[p.render() for p in self.parts],
+            *[p.render(specs) for p in self.parts],
             *([''] if self.kind == 'dir' else []),
         )
 
@@ -166,12 +210,21 @@ class DeployPath:
     def parse(cls, s: str) -> 'DeployPath':
         tail_parse: ta.Callable[[str], DeployPathPart]
         if s.endswith('/'):
-            tail_parse = DeployPathDir.parse
+            tail_parse = DirDeployPathPart.parse
             s = s[:-1]
         else:
-            tail_parse = DeployPathFile.parse
+            tail_parse = FileDeployPathPart.parse
         ps = check.non_empty_str(s).split('/')
         return cls([
-            *([DeployPathDir.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
+            *([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
             tail_parse(ps[-1]),
         ])
+
+
+##
+
+
+class DeployPathOwner(abc.ABC):
+    @abc.abstractmethod
+    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        raise NotImplementedError

ominfra/manage/deploy/types.py

@@ -0,0 +1,13 @@
+import typing as ta
+
+
+DeployHome = ta.NewType('DeployHome', str)
+
+DeployApp = ta.NewType('DeployApp', str)
+DeployTag = ta.NewType('DeployTag', str)
+DeployRev = ta.NewType('DeployRev', str)
+
+
+class DeployAppTag(ta.NamedTuple):
+    app: DeployApp
+    tag: DeployTag

ominfra/manage/deploy/venvs.py

@@ -0,0 +1,66 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - interp
+ - share more code with pyproject?
+"""
+import os.path
+import typing as ta
+
+from omlish.lite.asyncio.subprocesses import asyncio_subprocess_check_call
+
+from .paths import DeployPath
+from .paths import DeployPathOwner
+from .types import DeployAppTag
+from .types import DeployHome
+
+
+class DeployVenvManager(DeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: DeployHome,
+    ) -> None:
+        super().__init__()
+
+        self._deploy_home = deploy_home
+        self._dir = os.path.join(deploy_home, 'venvs')
+
+    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return {
+            DeployPath.parse('venvs/@app/@tag/'),
+        }
+
+    async def setup_venv(
+            self,
+            app_dir: str,
+            venv_dir: str,
+            *,
+            use_uv: bool = True,
+    ) -> None:
+        sys_exe = 'python3'
+
+        await asyncio_subprocess_check_call(sys_exe, '-m', 'venv', venv_dir)
+
+        #
+
+        venv_exe = os.path.join(venv_dir, 'bin', 'python3')
+
+        #
+
+        reqs_txt = os.path.join(app_dir, 'requirements.txt')
+
+        if os.path.isfile(reqs_txt):
+            if use_uv:
+                await asyncio_subprocess_check_call(venv_exe, '-m', 'pip', 'install', 'uv')
+                pip_cmd = ['-m', 'uv', 'pip']
+            else:
+                pip_cmd = ['-m', 'pip']
+
+            await asyncio_subprocess_check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
+
+    async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
+        await self.setup_venv(
+            os.path.join(self._deploy_home, 'apps', app_tag.app, app_tag.tag),
+            os.path.join(self._deploy_home, 'venvs', app_tag.app, app_tag.tag),
+        )