ominfra 0.0.0.dev166__tar.gz → 0.0.0.dev168__tar.gz

{ominfra-0.0.0.dev166/ominfra.egg-info → ominfra-0.0.0.dev168}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev166
+Version: 0.0.0.dev168
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev166
-Requires-Dist: omlish==0.0.0.dev166
+Requires-Dist: omdev==0.0.0.dev168
+Requires-Dist: omlish==0.0.0.dev168
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.18; extra == "all"

ominfra-0.0.0.dev168/ominfra/manage/deploy/apps.py

@@ -0,0 +1,175 @@
+# ruff: noqa: UP006 UP007
+import datetime
+import os.path
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.os.paths import relative_symlink
+
+from .conf import DeployConfManager
+from .git import DeployGitManager
+from .paths.owners import DeployPathOwner
+from .paths.paths import DeployPath
+from .specs import DeploySpec
+from .types import DeployAppTag
+from .types import DeployHome
+from .types import DeployKey
+from .types import DeployRev
+from .types import DeployTag
+from .venvs import DeployVenvManager
+
+
+def make_deploy_tag(
+        rev: DeployRev,
+        key: DeployKey,
+        *,
+        utcnow: ta.Optional[datetime.datetime] = None,
+) -> DeployTag:
+    if utcnow is None:
+        utcnow = datetime.datetime.now(tz=datetime.timezone.utc)  # noqa
+    now_fmt = '%Y%m%dT%H%M%SZ'
+    now_str = utcnow.strftime(now_fmt)
+    return DeployTag('-'.join([now_str, rev, key]))
+
+
+class DeployAppManager(DeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: ta.Optional[DeployHome] = None,
+
+            conf: DeployConfManager,
+            git: DeployGitManager,
+            venvs: DeployVenvManager,
+    ) -> None:
+        super().__init__()
+
+        self._deploy_home = deploy_home
+
+        self._conf = conf
+        self._git = git
+        self._venvs = venvs
+
+    #
+
+    _APP_TAG_DIR_STR = 'tags/apps/@app/@tag/'
+    _APP_TAG_DIR = DeployPath.parse(_APP_TAG_DIR_STR)
+
+    _CONF_TAG_DIR_STR = 'tags/conf/@tag--@app/'
+    _CONF_TAG_DIR = DeployPath.parse(_CONF_TAG_DIR_STR)
+
+    _DEPLOY_DIR_STR = 'deploys/@tag--@app/'
+    _DEPLOY_DIR = DeployPath.parse(_DEPLOY_DIR_STR)
+
+    _APP_DEPLOY_LINK = DeployPath.parse(f'{_DEPLOY_DIR_STR}apps/@app')
+    _CONF_DEPLOY_LINK = DeployPath.parse(f'{_DEPLOY_DIR_STR}conf')
+
+    @cached_nullary
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return {
+            self._APP_TAG_DIR,
+
+            self._CONF_TAG_DIR,
+
+            self._DEPLOY_DIR,
+
+            self._APP_DEPLOY_LINK,
+            self._CONF_DEPLOY_LINK,
+
+            *[
+                DeployPath.parse(f'{self._APP_TAG_DIR_STR}{sfx}/')
+                for sfx in [
+                    'conf',
+                    'git',
+                    'venv',
+                ]
+            ],
+        }
+
+    #
+
+    async def prepare_app(
+            self,
+            spec: DeploySpec,
+    ) -> None:
+        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.git.rev, spec.key()))
+
+        #
+
+        deploy_home = check.non_empty_str(self._deploy_home)
+
+        def build_path(pth: DeployPath) -> str:
+            return os.path.join(deploy_home, pth.render(app_tag.placeholders()))
+
+        app_tag_dir = build_path(self._APP_TAG_DIR)
+        conf_tag_dir = build_path(self._CONF_TAG_DIR)
+        deploy_dir = build_path(self._DEPLOY_DIR)
+        app_deploy_link = build_path(self._APP_DEPLOY_LINK)
+        conf_deploy_link_file = build_path(self._CONF_DEPLOY_LINK)
+
+        #
+
+        os.makedirs(deploy_dir)
+
+        deploying_link = os.path.join(deploy_home, 'deploys/deploying')
+        relative_symlink(
+            deploy_dir,
+            deploying_link,
+            target_is_directory=True,
+            make_dirs=True,
+        )
+
+        #
+
+        os.makedirs(app_tag_dir)
+        relative_symlink(
+            app_tag_dir,
+            app_deploy_link,
+            target_is_directory=True,
+            make_dirs=True,
+        )
+
+        #
+
+        os.makedirs(conf_tag_dir)
+        relative_symlink(
+            conf_tag_dir,
+            conf_deploy_link_file,
+            target_is_directory=True,
+            make_dirs=True,
+        )
+
+        #
+
+        git_dir = os.path.join(app_tag_dir, 'git')
+        await self._git.checkout(
+            spec.git,
+            git_dir,
+        )
+
+        #
+
+        if spec.venv is not None:
+            venv_dir = os.path.join(app_tag_dir, 'venv')
+            await self._venvs.setup_venv(
+                spec.venv,
+                git_dir,
+                venv_dir,
+            )
+
+        #
+
+        if spec.conf is not None:
+            conf_dir = os.path.join(app_tag_dir, 'conf')
+            await self._conf.write_conf(
+                spec.conf,
+                app_tag,
+                conf_dir,
+                conf_tag_dir,
+            )
+
+        #
+
+        current_link = os.path.join(deploy_home, 'deploys/current')
+        os.replace(deploying_link, current_link)

{ominfra-0.0.0.dev166 → ominfra-0.0.0.dev168}/ominfra/manage/deploy/commands.py

@@ -5,7 +5,7 @@ from omlish.lite.logs import log
 
 from ..commands.base import Command
 from ..commands.base import CommandExecutor
-from .apps import DeployAppManager
+from .deploy import DeployManager
 from .specs import DeploySpec
 
 
@@ -23,11 +23,11 @@ class DeployCommand(Command['DeployCommand.Output']):
 
 @dc.dataclass(frozen=True)
 class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
-    _apps: DeployAppManager
+    _deploy: DeployManager
 
     async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
         log.info('Deploying! %r', cmd.spec)
 
-        await self._apps.prepare_app(cmd.spec)
+        await self._deploy.run_deploy(cmd.spec)
 
         return DeployCommand.Output()

ominfra-0.0.0.dev168/ominfra/manage/deploy/conf.py

@@ -0,0 +1,183 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - @conf DeployPathPlaceholder? :|
+ - post-deploy: remove any dir_links not present in new spec
+  - * only if succeeded * - otherwise, remove any dir_links present in new spec but not previously present?
+   - no such thing as 'previously present'.. build a 'deploy state' and pass it back?
+ - ** whole thing can be atomic **
+  - 1) new atomic temp dir
+  - 2) for each subdir not needing modification, hardlink into temp dir
+  - 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
+  - 4) write (or if deleting, omit) new files
+  - 5) swap top level
+ - ** whole deploy can be atomic(-ish) - do this for everything **
+  - just a '/deploy/current' dir
+  - some things (venvs) cannot be moved, thus the /deploy/venvs dir
+  - ** ensure (enforce) equivalent relpath nesting
+"""
+import os.path
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.os.paths import is_path_in_dir
+from omlish.os.paths import relative_symlink
+
+from .paths.paths import DEPLOY_PATH_PLACEHOLDER_SEPARATOR
+from .specs import AppDeployConfLink
+from .specs import DeployConfFile
+from .specs import DeployConfLink
+from .specs import DeployConfSpec
+from .specs import TagDeployConfLink
+from .types import DeployAppTag
+from .types import DeployHome
+
+
+class DeployConfManager:
+    def __init__(
+            self,
+            *,
+            deploy_home: ta.Optional[DeployHome] = None,
+    ) -> None:
+        super().__init__()
+
+        self._deploy_home = deploy_home
+
+    #
+
+    async def _write_conf_file(
+            self,
+            cf: DeployConfFile,
+            conf_dir: str,
+    ) -> None:
+        conf_file = os.path.join(conf_dir, cf.path)
+        check.arg(is_path_in_dir(conf_dir, conf_file))
+
+        os.makedirs(os.path.dirname(conf_file), exist_ok=True)
+
+        with open(conf_file, 'w') as f:  # noqa
+            f.write(cf.body)
+
+    #
+
+    class _ComputedConfLink(ta.NamedTuple):
+        is_dir: bool
+        link_src: str
+        link_dst: str
+
+    def _compute_conf_link_dst(
+            self,
+            link: DeployConfLink,
+            app_tag: DeployAppTag,
+            conf_dir: str,
+            link_dir: str,
+    ) -> _ComputedConfLink:
+        link_src = os.path.join(conf_dir, link.src)
+        check.arg(is_path_in_dir(conf_dir, link_src))
+
+        #
+
+        if (is_dir := link.src.endswith('/')):
+            # @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
+            check.arg(link.src.count('/') == 1)
+            link_dst_pfx = link.src
+            link_dst_sfx = ''
+
+        elif '/' in link.src:
+            # @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
+            d, f = os.path.split(link.src)
+            # TODO: check filename :|
+            link_dst_pfx = d + '/'
+            link_dst_sfx = DEPLOY_PATH_PLACEHOLDER_SEPARATOR + f
+
+        else:  # noqa
+            # @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
+            if '.' in link.src:
+                l, _, r = link.src.partition('.')
+                link_dst_pfx = l + '/'
+                link_dst_sfx = '.' + r
+            else:
+                link_dst_pfx = link.src + '/'
+                link_dst_sfx = ''
+
+        #
+
+        if isinstance(link, AppDeployConfLink):
+            link_dst_mid = str(app_tag.app)
+        elif isinstance(link, TagDeployConfLink):
+            link_dst_mid = DEPLOY_PATH_PLACEHOLDER_SEPARATOR.join([app_tag.app, app_tag.tag])
+        else:
+            raise TypeError(link)
+
+        #
+
+        link_dst_name = ''.join([
+            link_dst_pfx,
+            link_dst_mid,
+            link_dst_sfx,
+        ])
+        link_dst = os.path.join(link_dir, link_dst_name)
+
+        return DeployConfManager._ComputedConfLink(
+            is_dir=is_dir,
+            link_src=link_src,
+            link_dst=link_dst,
+        )
+
+    async def _make_conf_link(
+            self,
+            link: DeployConfLink,
+            app_tag: DeployAppTag,
+            conf_dir: str,
+            link_dir: str,
+    ) -> None:
+        comp = self._compute_conf_link_dst(
+            link,
+            app_tag,
+            conf_dir,
+            link_dir,
+        )
+
+        #
+
+        check.arg(is_path_in_dir(conf_dir, comp.link_src))
+        check.arg(is_path_in_dir(link_dir, comp.link_dst))
+
+        if comp.is_dir:
+            check.arg(os.path.isdir(comp.link_src))
+        else:
+            check.arg(os.path.isfile(comp.link_src))
+
+        #
+
+        relative_symlink(  # noqa
+            comp.link_src,
+            comp.link_dst,
+            target_is_directory=comp.is_dir,
+            make_dirs=True,
+        )
+
+    #
+
+    async def write_conf(
+            self,
+            spec: DeployConfSpec,
+            app_tag: DeployAppTag,
+            conf_dir: str,
+            link_dir: str,
+    ) -> None:
+        for cf in spec.files or []:
+            await self._write_conf_file(
+                cf,
+                conf_dir,
+            )
+
+        #
+
+        for link in spec.links or []:
+            await self._make_conf_link(
+                link,
+                app_tag,
+                conf_dir,
+                link_dir,
+            )

ominfra-0.0.0.dev168/ominfra/manage/deploy/deploy.py

@@ -0,0 +1,27 @@
+# ruff: noqa: UP006 UP007
+from .apps import DeployAppManager
+from .paths.manager import DeployPathsManager
+from .specs import DeploySpec
+
+
+class DeployManager:
+    def __init__(
+            self,
+            *,
+            apps: DeployAppManager,
+            paths: DeployPathsManager,
+    ):
+        super().__init__()
+
+        self._apps = apps
+        self._paths = paths
+
+    async def run_deploy(
+            self,
+            spec: DeploySpec,
+    ) -> None:
+        self._paths.validate_deploy_paths()
+
+        #
+
+        await self._apps.prepare_app(spec)

{ominfra-0.0.0.dev166 → ominfra-0.0.0.dev168}/ominfra/manage/deploy/git.py

@@ -17,9 +17,9 @@ from omlish.lite.cached import async_cached_nullary
 from omlish.lite.check import check
 from omlish.os.atomics import AtomicPathSwapping
 
-from .paths import SingleDirDeployPathOwner
-from .specs import DeployGitCheckout
+from .paths.owners import SingleDirDeployPathOwner
 from .specs import DeployGitRepo
+from .specs import DeployGitSpec
 from .types import DeployHome
 from .types import DeployRev
 
@@ -95,7 +95,7 @@ class DeployGitManager(SingleDirDeployPathOwner):
 
         #
 
-        async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
+        async def checkout(self, spec: DeployGitSpec, dst_dir: str) -> None:
             check.state(not os.path.exists(dst_dir))
             with self._git._atomics.begin_atomic_path_swap(  # noqa
                     'dir',
@@ -103,14 +103,14 @@ class DeployGitManager(SingleDirDeployPathOwner):
                     auto_commit=True,
                     make_dirs=True,
             ) as dst_swap:
-                await self.fetch(checkout.rev)
+                await self.fetch(spec.rev)
 
                 dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
                 await dst_call('git', 'init')
 
                 await dst_call('git', 'remote', 'add', 'local', self._dir)
-                await dst_call('git', 'fetch', '--depth=1', 'local', checkout.rev)
-                await dst_call('git', 'checkout', checkout.rev, *(checkout.subtrees or []))
+                await dst_call('git', 'fetch', '--depth=1', 'local', spec.rev)
+                await dst_call('git', 'checkout', spec.rev, *(spec.subtrees or []))
 
     def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
         try:
@@ -119,5 +119,9 @@ class DeployGitManager(SingleDirDeployPathOwner):
             repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
             return repo_dir
 
-    async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
-        await self.get_repo_dir(checkout.repo).checkout(checkout, dst_dir)
+    async def checkout(
+            self,
+            spec: DeployGitSpec,
+            dst_dir: str,
+    ) -> None:
+        await self.get_repo_dir(spec.repo).checkout(spec, dst_dir)

{ominfra-0.0.0.dev166 → ominfra-0.0.0.dev168}/ominfra/manage/deploy/inject.py

@@ -11,10 +11,14 @@ from ..commands.inject import bind_command
 from .apps import DeployAppManager
 from .commands import DeployCommand
 from .commands import DeployCommandExecutor
+from .conf import DeployConfManager
 from .config import DeployConfig
+from .deploy import DeployManager
 from .git import DeployGitManager
 from .interp import InterpCommand
 from .interp import InterpCommandExecutor
+from .paths.inject import bind_deploy_paths
+from .paths.owners import DeployPathOwner
 from .tmp import DeployTmpManager
 from .types import DeployHome
 from .venvs import DeployVenvManager
@@ -27,22 +31,43 @@ def bind_deploy(
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(deploy_config),
 
-        #
+        bind_deploy_paths(),
+    ]
+
+    #
+
+    def bind_manager(cls: type) -> InjectorBindings:
+        return inj.as_bindings(
+            inj.bind(cls, singleton=True),
+
+            *([inj.bind(DeployPathOwner, to_key=cls, array=True)] if issubclass(cls, DeployPathOwner) else []),
+        )
+
+    #
 
-        inj.bind(DeployAppManager, singleton=True),
+    lst.extend([
+        bind_manager(DeployAppManager),
 
-        inj.bind(DeployGitManager, singleton=True),
+        bind_manager(DeployConfManager),
 
-        inj.bind(DeployTmpManager, singleton=True),
+        bind_manager(DeployGitManager),
+
+        bind_manager(DeployManager),
+
+        bind_manager(DeployTmpManager),
         inj.bind(AtomicPathSwapping, to_key=DeployTmpManager),
 
-        inj.bind(DeployVenvManager, singleton=True),
+        bind_manager(DeployVenvManager),
+    ])
 
-        #
+    #
 
+    lst.extend([
         bind_command(DeployCommand, DeployCommandExecutor),
         bind_command(InterpCommand, InterpCommandExecutor),
-    ]
+    ])
+
+    #
 
     if (dh := deploy_config.deploy_home) is not None:
         dh = os.path.abspath(os.path.expanduser(dh))

ominfra-0.0.0.dev168/ominfra/manage/deploy/paths/inject.py

@@ -0,0 +1,21 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
+
+from .manager import DeployPathsManager
+from .owners import DeployPathOwner
+from .owners import DeployPathOwners
+
+
+def bind_deploy_paths() -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = [
+        inj.bind_array(DeployPathOwner),
+        inj.bind_array_type(DeployPathOwner, DeployPathOwners),
+
+        inj.bind(DeployPathsManager, singleton=True),
+    ]
+
+    return inj.as_bindings(*lst)

ominfra-0.0.0.dev168/ominfra/manage/deploy/paths/manager.py

@@ -0,0 +1,36 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+
+from ..types import DeployHome
+from .owners import DeployPathOwner
+from .owners import DeployPathOwners
+from .paths import DeployPath
+from .paths import DeployPathError
+
+
+class DeployPathsManager:
+    def __init__(
+            self,
+            *,
+            deploy_home: ta.Optional[DeployHome],
+            deploy_path_owners: DeployPathOwners,
+    ) -> None:
+        super().__init__()
+
+        self._deploy_home = deploy_home
+        self._deploy_path_owners = deploy_path_owners
+
+    @cached_nullary
+    def owners_by_path(self) -> ta.Mapping[DeployPath, DeployPathOwner]:
+        dct: ta.Dict[DeployPath, DeployPathOwner] = {}
+        for o in self._deploy_path_owners:
+            for p in o.get_owned_deploy_paths():
+                if p in dct:
+                    raise DeployPathError(f'Duplicate deploy path owner: {p}')
+                dct[p] = o
+        return dct
+
+    def validate_deploy_paths(self) -> None:
+        self.owners_by_path()

ominfra-0.0.0.dev168/ominfra/manage/deploy/paths/owners.py

@@ -0,0 +1,50 @@
+# ruff: noqa: UP006 UP007
+import abc
+import os.path
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+
+from ..types import DeployHome
+from .paths import DeployPath
+
+
+class DeployPathOwner(abc.ABC):
+    @abc.abstractmethod
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        raise NotImplementedError
+
+
+DeployPathOwners = ta.NewType('DeployPathOwners', ta.Sequence[DeployPathOwner])
+
+
+class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
+    def __init__(
+            self,
+            *args: ta.Any,
+            owned_dir: str,
+            deploy_home: ta.Optional[DeployHome],
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+
+        check.not_in('/', owned_dir)
+        self._owned_dir: str = check.non_empty_str(owned_dir)
+
+        self._deploy_home = deploy_home
+
+        self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
+
+    @cached_nullary
+    def _dir(self) -> str:
+        return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
+
+    @cached_nullary
+    def _make_dir(self) -> str:
+        if not os.path.isdir(d := self._dir()):
+            os.makedirs(d, exist_ok=True)
+        return d
+
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return self._owned_deploy_paths