ominfra 0.0.0.dev154__tar.gz → 0.0.0.dev156__tar.gz

{ominfra-0.0.0.dev154/ominfra.egg-info → ominfra-0.0.0.dev156}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev154
+Version: 0.0.0.dev156
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev154
-Requires-Dist: omlish==0.0.0.dev154
+Requires-Dist: omdev==0.0.0.dev156
+Requires-Dist: omlish==0.0.0.dev156
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.18; extra == "all"

{ominfra-0.0.0.dev154 → ominfra-0.0.0.dev156}/ominfra/manage/bootstrap.py

@@ -1,6 +1,8 @@
+# ruff: noqa: UP006 UP007
 import dataclasses as dc
 
 from .config import MainConfig
+from .deploy.config import DeployConfig
 from .remote.config import RemoteConfig
 from .system.config import SystemConfig
 
@@ -9,6 +11,8 @@ from .system.config import SystemConfig
 class MainBootstrap:
     main_config: MainConfig = MainConfig()
 
+    deploy_config: DeployConfig = DeployConfig()
+
     remote_config: RemoteConfig = RemoteConfig()
 
     system_config: SystemConfig = SystemConfig()

{ominfra-0.0.0.dev154 → ominfra-0.0.0.dev156}/ominfra/manage/bootstrap_.py

@@ -1,3 +1,4 @@
+# ruff: noqa: UP006 UP007
 from omlish.lite.inject import Injector
 from omlish.lite.inject import inj
 from omlish.lite.logs import configure_standard_logging
@@ -12,8 +13,12 @@ def main_bootstrap(bs: MainBootstrap) -> Injector:
 
     injector = inj.create_injector(bind_main(  # noqa
         main_config=bs.main_config,
+
+        deploy_config=bs.deploy_config,
         remote_config=bs.remote_config,
         system_config=bs.system_config,
+
+        main_bootstrap=bs,
     ))
 
     return injector

{ominfra-0.0.0.dev154 → ominfra-0.0.0.dev156}/ominfra/manage/commands/inject.py

@@ -18,13 +18,13 @@ from .base import CommandNameMap
 from .base import CommandRegistration
 from .base import CommandRegistrations
 from .base import build_command_name_map
-from .execution import CommandExecutorMap
-from .execution import LocalCommandExecutor
-from .interp import InterpCommand
-from .interp import InterpCommandExecutor
+from .local import LocalCommandExecutor
 from .marshal import install_command_marshaling
+from .ping import PingCommand
+from .ping import PingCommandExecutor
 from .subprocess import SubprocessCommand
 from .subprocess import SubprocessCommandExecutor
+from .types import CommandExecutorMap
 
 
 ##
@@ -113,13 +113,10 @@ def bind_commands(
 
     #
 
-    command_cls: ta.Any
-    executor_cls: ta.Any
-    for command_cls, executor_cls in [
-        (SubprocessCommand, SubprocessCommandExecutor),
-        (InterpCommand, InterpCommandExecutor),
-    ]:
-        lst.append(bind_command(command_cls, executor_cls))
+    lst.extend([
+        bind_command(PingCommand, PingCommandExecutor),
+        bind_command(SubprocessCommand, SubprocessCommandExecutor),
+    ])
 
     #
 

ominfra-0.0.0.dev154/ominfra/manage/commands/execution.py → ominfra-0.0.0.dev156/ominfra/manage/commands/local.py

@@ -1,11 +1,7 @@
 # ruff: noqa: UP006 UP007
-import typing as ta
-
 from .base import Command
 from .base import CommandExecutor
-
-
-CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
+from .types import CommandExecutorMap
 
 
 class LocalCommandExecutor(CommandExecutor):

ominfra-0.0.0.dev156/ominfra/manage/commands/ping.py

@@ -0,0 +1,23 @@
+# ruff: noqa: TC003 UP006 UP007
+import dataclasses as dc
+import time
+
+from .base import Command
+from .base import CommandExecutor
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class PingCommand(Command['PingCommand.Output']):
+    time: float = dc.field(default_factory=time.time)
+
+    @dc.dataclass(frozen=True)
+    class Output(Command.Output):
+        time: float
+
+
+class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
+    async def execute(self, cmd: PingCommand) -> PingCommand.Output:
+        return PingCommand.Output(cmd.time)

{ominfra-0.0.0.dev154 → ominfra-0.0.0.dev156}/ominfra/manage/commands/subprocess.py

@@ -6,8 +6,7 @@ import subprocess
 import time
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_communicate
-from omlish.lite.asyncio.subprocesses import asyncio_subprocess_popen
+from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
 from omlish.lite.subprocesses import SubprocessChannelOption
@@ -51,7 +50,7 @@ class SubprocessCommand(Command['SubprocessCommand.Output']):
 class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
     async def execute(self, cmd: SubprocessCommand) -> SubprocessCommand.Output:
         proc: asyncio.subprocess.Process
-        async with asyncio_subprocess_popen(
+        async with asyncio_subprocesses.popen(
             *subprocess_maybe_shell_wrap_exec(*cmd.cmd),
 
             shell=cmd.shell,
@@ -65,7 +64,7 @@ class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCom
             timeout=cmd.timeout,
         ) as proc:
             start_time = time.time()
-            stdout, stderr = await asyncio_subprocess_communicate(
+            stdout, stderr = await asyncio_subprocesses.communicate(
                 proc,
                 input=cmd.input,
                 timeout=cmd.timeout,

ominfra-0.0.0.dev156/ominfra/manage/commands/types.py

@@ -0,0 +1,8 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from .base import Command
+from .base import CommandExecutor
+
+
+CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])

ominfra-0.0.0.dev156/ominfra/manage/deploy/apps.py

@@ -0,0 +1,72 @@
+# ruff: noqa: UP006 UP007
+import datetime
+import os.path
+import typing as ta
+
+from .git import DeployGitManager
+from .git import DeployGitRepo
+from .git import DeployGitSpec
+from .paths import DeployPath
+from .paths import DeployPathOwner
+from .types import DeployApp
+from .types import DeployAppTag
+from .types import DeployHome
+from .types import DeployRev
+from .types import DeployTag
+from .venvs import DeployVenvManager
+
+
+def make_deploy_tag(
+        rev: DeployRev,
+        now: ta.Optional[datetime.datetime] = None,
+) -> DeployTag:
+    if now is None:
+        now = datetime.datetime.utcnow()  # noqa
+    now_fmt = '%Y%m%dT%H%M%S'
+    now_str = now.strftime(now_fmt)
+    return DeployTag('-'.join([rev, now_str]))
+
+
+class DeployAppManager(DeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: DeployHome,
+            git: DeployGitManager,
+            venvs: DeployVenvManager,
+    ) -> None:
+        super().__init__()
+
+        self._deploy_home = deploy_home
+        self._git = git
+        self._venvs = venvs
+
+        self._dir = os.path.join(deploy_home, 'apps')
+
+    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return {
+            DeployPath.parse('apps/@app/@tag'),
+        }
+
+    async def prepare_app(
+            self,
+            app: DeployApp,
+            rev: DeployRev,
+            repo: DeployGitRepo,
+    ):
+        app_tag = DeployAppTag(app, make_deploy_tag(rev))
+        app_dir = os.path.join(self._dir, app, app_tag.tag)
+
+        #
+
+        await self._git.checkout(
+            DeployGitSpec(
+                repo=repo,
+                rev=rev,
+            ),
+            app_dir,
+        )
+
+        #
+
+        await self._venvs.setup_app_venv(app_tag)

{ominfra-0.0.0.dev154/ominfra/manage/system → ominfra-0.0.0.dev156/ominfra/manage/deploy}/config.py

@@ -4,5 +4,5 @@ import typing as ta
 
 
 @dc.dataclass(frozen=True)
-class SystemConfig:
-    platform: ta.Optional[str] = None
+class DeployConfig:
+    deploy_home: ta.Optional[str] = None

ominfra-0.0.0.dev156/ominfra/manage/deploy/git.py

@@ -0,0 +1,136 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - 'repos'?
+
+git/github.com/wrmsr/omlish <- bootstrap repo
+ - shallow clone off bootstrap into /apps
+
+github.com/wrmsr/omlish@rev
+"""
+import dataclasses as dc
+import functools
+import os.path
+import typing as ta
+
+from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
+from omlish.lite.cached import async_cached_nullary
+from omlish.lite.check import check
+
+from .paths import DeployPath
+from .paths import DeployPathOwner
+from .types import DeployHome
+from .types import DeployRev
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployGitRepo:
+    host: ta.Optional[str] = None
+    username: ta.Optional[str] = None
+    path: ta.Optional[str] = None
+
+    def __post_init__(self) -> None:
+        check.not_in('..', check.non_empty_str(self.host))
+        check.not_in('.', check.non_empty_str(self.path))
+
+
+@dc.dataclass(frozen=True)
+class DeployGitSpec:
+    repo: DeployGitRepo
+    rev: DeployRev
+
+
+##
+
+
+class DeployGitManager(DeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: DeployHome,
+    ) -> None:
+        super().__init__()
+
+        self._deploy_home = deploy_home
+        self._dir = os.path.join(deploy_home, 'git')
+
+        self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
+
+    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return {
+            DeployPath.parse('git'),
+        }
+
+    class RepoDir:
+        def __init__(
+                self,
+                git: 'DeployGitManager',
+                repo: DeployGitRepo,
+        ) -> None:
+            super().__init__()
+
+            self._git = git
+            self._repo = repo
+            self._dir = os.path.join(
+                self._git._dir,  # noqa
+                check.non_empty_str(repo.host),
+                check.non_empty_str(repo.path),
+            )
+
+        @property
+        def repo(self) -> DeployGitRepo:
+            return self._repo
+
+        @property
+        def url(self) -> str:
+            if self._repo.username is not None:
+                return f'{self._repo.username}@{self._repo.host}:{self._repo.path}'
+            else:
+                return f'https://{self._repo.host}/{self._repo.path}'
+
+        async def _call(self, *cmd: str) -> None:
+            await asyncio_subprocesses.check_call(
+                *cmd,
+                cwd=self._dir,
+            )
+
+        @async_cached_nullary
+        async def init(self) -> None:
+            os.makedirs(self._dir, exist_ok=True)
+            if os.path.exists(os.path.join(self._dir, '.git')):
+                return
+
+            await self._call('git', 'init')
+            await self._call('git', 'remote', 'add', 'origin', self.url)
+
+        async def fetch(self, rev: DeployRev) -> None:
+            await self.init()
+            await self._call('git', 'fetch', '--depth=1', 'origin', rev)
+
+        async def checkout(self, rev: DeployRev, dst_dir: str) -> None:
+            check.state(not os.path.exists(dst_dir))
+
+            await self.fetch(rev)
+
+            # FIXME: temp dir swap
+            os.makedirs(dst_dir)
+
+            dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_dir)
+            await dst_call('git', 'init')
+
+            await dst_call('git', 'remote', 'add', 'local', self._dir)
+            await dst_call('git', 'fetch', '--depth=1', 'local', rev)
+            await dst_call('git', 'checkout', rev)
+
+    def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
+        try:
+            return self._repo_dirs[repo]
+        except KeyError:
+            repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
+            return repo_dir
+
+    async def checkout(self, spec: DeployGitSpec, dst_dir: str) -> None:
+        await self.get_repo_dir(spec.repo).checkout(spec.rev, dst_dir)

ominfra-0.0.0.dev156/ominfra/manage/deploy/inject.py

@@ -0,0 +1,40 @@
+# ruff: noqa: UP006 UP007
+import os.path
+import typing as ta
+
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
+
+from ..commands.inject import bind_command
+from .apps import DeployAppManager
+from .commands import DeployCommand
+from .commands import DeployCommandExecutor
+from .config import DeployConfig
+from .git import DeployGitManager
+from .interp import InterpCommand
+from .interp import InterpCommandExecutor
+from .types import DeployHome
+from .venvs import DeployVenvManager
+
+
+def bind_deploy(
+        *,
+        deploy_config: DeployConfig,
+) -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = [
+        inj.bind(deploy_config),
+
+        inj.bind(DeployAppManager, singleton=True),
+        inj.bind(DeployGitManager, singleton=True),
+        inj.bind(DeployVenvManager, singleton=True),
+
+        bind_command(DeployCommand, DeployCommandExecutor),
+        bind_command(InterpCommand, InterpCommandExecutor),
+    ]
+
+    if (dh := deploy_config.deploy_home) is not None:
+        dh = os.path.abspath(os.path.expanduser(dh))
+        lst.append(inj.bind(dh, key=DeployHome))
+
+    return inj.as_bindings(*lst)

ominfra-0.0.0.dev156/ominfra/manage/deploy/paths.py

@@ -0,0 +1,230 @@
+# ruff: noqa: UP006 UP007
+"""
+~deploy
+  deploy.pid (flock)
+  /app
+    /<appspec> - shallow clone
+  /conf
+    /env
+      <appspec>.env
+    /nginx
+      <appspec>.conf
+    /supervisor
+      <appspec>.conf
+  /venv
+    /<appspec>
+
+?
+  /logs
+    /wrmsr--omlish--<spec>
+
+spec = <name>--<rev>--<when>
+
+==
+
+for dn in [
+    'app',
+    'conf',
+    'conf/env',
+    'conf/nginx',
+    'conf/supervisor',
+    'venv',
+]:
+
+==
+
+"""
+import abc
+import dataclasses as dc
+import os.path
+import typing as ta
+
+from omlish.lite.check import check
+
+
+DeployPathKind = ta.Literal['dir', 'file']  # ta.TypeAlias
+DeployPathSpec = ta.Literal['app', 'tag']  # ta.TypeAlias
+
+
+##
+
+
+DEPLOY_PATH_SPEC_PLACEHOLDER = '@'
+DEPLOY_PATH_SPEC_SEPARATORS = '-.'
+
+DEPLOY_PATH_SPECS: ta.FrozenSet[str] = frozenset([
+    'app',
+    'tag',  # <rev>-<dt>
+])
+
+
+class DeployPathError(Exception):
+    pass
+
+
+@dc.dataclass(frozen=True)
+class DeployPathPart(abc.ABC):  # noqa
+    @property
+    @abc.abstractmethod
+    def kind(self) -> DeployPathKind:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+        raise NotImplementedError
+
+
+#
+
+
+class DirDeployPathPart(DeployPathPart, abc.ABC):
+    @property
+    def kind(self) -> DeployPathKind:
+        return 'dir'
+
+    @classmethod
+    def parse(cls, s: str) -> 'DirDeployPathPart':
+        if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
+            check.equal(s[0], DEPLOY_PATH_SPEC_PLACEHOLDER)
+            return SpecDirDeployPathPart(s[1:])
+        else:
+            return ConstDirDeployPathPart(s)
+
+
+class FileDeployPathPart(DeployPathPart, abc.ABC):
+    @property
+    def kind(self) -> DeployPathKind:
+        return 'file'
+
+    @classmethod
+    def parse(cls, s: str) -> 'FileDeployPathPart':
+        if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
+            check.equal(s[0], DEPLOY_PATH_SPEC_PLACEHOLDER)
+            if not any(c in s for c in DEPLOY_PATH_SPEC_SEPARATORS):
+                return SpecFileDeployPathPart(s[1:], '')
+            else:
+                p = min(f for c in DEPLOY_PATH_SPEC_SEPARATORS if (f := s.find(c)) > 0)
+                return SpecFileDeployPathPart(s[1:p], s[p:])
+        else:
+            return ConstFileDeployPathPart(s)
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class ConstDeployPathPart(DeployPathPart, abc.ABC):
+    name: str
+
+    def __post_init__(self) -> None:
+        check.non_empty_str(self.name)
+        check.not_in('/', self.name)
+        check.not_in(DEPLOY_PATH_SPEC_PLACEHOLDER, self.name)
+
+    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+        return self.name
+
+
+class ConstDirDeployPathPart(ConstDeployPathPart, DirDeployPathPart):
+    pass
+
+
+class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
+    pass
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class SpecDeployPathPart(DeployPathPart, abc.ABC):
+    spec: str  # DeployPathSpec
+
+    def __post_init__(self) -> None:
+        check.non_empty_str(self.spec)
+        for c in [*DEPLOY_PATH_SPEC_SEPARATORS, DEPLOY_PATH_SPEC_PLACEHOLDER, '/']:
+            check.not_in(c, self.spec)
+        check.in_(self.spec, DEPLOY_PATH_SPECS)
+
+    def _render_spec(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+        if specs is not None:
+            return specs[self.spec]  # type: ignore
+        else:
+            return DEPLOY_PATH_SPEC_PLACEHOLDER + self.spec
+
+
+@dc.dataclass(frozen=True)
+class SpecDirDeployPathPart(SpecDeployPathPart, DirDeployPathPart):
+    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+        return self._render_spec(specs)
+
+
+@dc.dataclass(frozen=True)
+class SpecFileDeployPathPart(SpecDeployPathPart, FileDeployPathPart):
+    suffix: str
+
+    def __post_init__(self) -> None:
+        super().__post_init__()
+        if self.suffix:
+            for c in [DEPLOY_PATH_SPEC_PLACEHOLDER, '/']:
+                check.not_in(c, self.suffix)
+
+    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+        return self._render_spec(specs) + self.suffix
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployPath:
+    parts: ta.Sequence[DeployPathPart]
+
+    def __post_init__(self) -> None:
+        check.not_empty(self.parts)
+        for p in self.parts[:-1]:
+            check.equal(p.kind, 'dir')
+
+        pd = {}
+        for i, p in enumerate(self.parts):
+            if isinstance(p, SpecDeployPathPart):
+                if p.spec in pd:
+                    raise DeployPathError('Duplicate specs in path', self)
+                pd[p.spec] = i
+
+        if 'tag' in pd:
+            if 'app' not in pd or pd['app'] >= pd['tag']:
+                raise DeployPathError('Tag spec in path without preceding app', self)
+
+    @property
+    def kind(self) -> ta.Literal['file', 'dir']:
+        return self.parts[-1].kind
+
+    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+        return os.path.join(  # noqa
+            *[p.render(specs) for p in self.parts],
+            *([''] if self.kind == 'dir' else []),
+        )
+
+    @classmethod
+    def parse(cls, s: str) -> 'DeployPath':
+        tail_parse: ta.Callable[[str], DeployPathPart]
+        if s.endswith('/'):
+            tail_parse = DirDeployPathPart.parse
+            s = s[:-1]
+        else:
+            tail_parse = FileDeployPathPart.parse
+        ps = check.non_empty_str(s).split('/')
+        return cls([
+            *([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
+            tail_parse(ps[-1]),
+        ])
+
+
+##
+
+
+class DeployPathOwner(abc.ABC):
+    @abc.abstractmethod
+    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        raise NotImplementedError

ominfra-0.0.0.dev156/ominfra/manage/deploy/types.py

@@ -0,0 +1,13 @@
+import typing as ta
+
+
+DeployHome = ta.NewType('DeployHome', str)
+
+DeployApp = ta.NewType('DeployApp', str)
+DeployTag = ta.NewType('DeployTag', str)
+DeployRev = ta.NewType('DeployRev', str)
+
+
+class DeployAppTag(ta.NamedTuple):
+    app: DeployApp
+    tag: DeployTag