ominfra 0.0.0.dev166__py3-none-any.whl → 0.0.0.dev167__py3-none-any.whl

ominfra/manage/deploy/apps.py

@@ -5,7 +5,9 @@ import typing as ta
 
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
+from omlish.os.paths import relative_symlink
 
+from .conf import DeployConfManager
 from .git import DeployGitManager
 from .paths import DeployPath
 from .paths import DeployPathOwner
@@ -36,12 +38,16 @@ class DeployAppManager(DeployPathOwner):
             self,
             *,
             deploy_home: ta.Optional[DeployHome] = None,
+
+            conf: DeployConfManager,
             git: DeployGitManager,
             venvs: DeployVenvManager,
     ) -> None:
         super().__init__()
 
         self._deploy_home = deploy_home
+
+        self._conf = conf
         self._git = git
         self._venvs = venvs
 
@@ -51,24 +57,69 @@ class DeployAppManager(DeployPathOwner):
 
     def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
         return {
-            DeployPath.parse('apps/@app/@tag/'),
+            DeployPath.parse('apps/@app/current'),
+            DeployPath.parse('apps/@app/deploying'),
+
+            DeployPath.parse('apps/@app/tags/@tag/conf/'),
+            DeployPath.parse('apps/@app/tags/@tag/git/'),
+            DeployPath.parse('apps/@app/tags/@tag/venv/'),
         }
 
     async def prepare_app(
             self,
             spec: DeploySpec,
     ) -> None:
-        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.checkout.rev, spec.key()))
-        app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
+        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.git.rev, spec.key()))
+
+        #
+
+        app_dir = os.path.join(self._dir(), spec.app)
+        os.makedirs(app_dir, exist_ok=True)
 
         #
 
+        tag_dir = os.path.join(app_dir, 'tags', app_tag.tag)
+        os.makedirs(tag_dir)
+
+        #
+
+        deploying_file = os.path.join(app_dir, 'deploying')
+        current_file = os.path.join(app_dir, 'current')
+
+        if os.path.exists(deploying_file):
+            os.unlink(deploying_file)
+        relative_symlink(tag_dir, deploying_file, target_is_directory=True)
+
+        #
+
+        git_dir = os.path.join(tag_dir, 'git')
         await self._git.checkout(
-            spec.checkout,
-            app_dir,
+            spec.git,
+            git_dir,
         )
 
         #
 
         if spec.venv is not None:
-            await self._venvs.setup_app_venv(app_tag, spec.venv)
+            venv_dir = os.path.join(tag_dir, 'venv')
+            await self._venvs.setup_venv(
+                spec.venv,
+                git_dir,
+                venv_dir,
+            )
+
+        #
+
+        if spec.conf is not None:
+            conf_dir = os.path.join(tag_dir, 'conf')
+            conf_link_dir = os.path.join(current_file, 'conf')
+            await self._conf.write_conf(
+                spec.conf,
+                conf_dir,
+                app_tag,
+                conf_link_dir,
+            )
+
+        #
+
+        os.replace(deploying_file, current_file)

ominfra/manage/deploy/conf.py

@@ -0,0 +1,143 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - @conf DeployPathPlaceholder? :|
+ - post-deploy: remove any dir_links not present in new spec
+  - * only if succeeded * - otherwise, remove any dir_links present in new spec but not previously present?
+   - no such thing as 'previously present'.. build a 'deploy state' and pass it back?
+ - ** whole thing can be atomic **
+  - 1) new atomic temp dir
+  - 2) for each subdir not needing modification, hardlink into temp dir
+  - 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
+  - 4) write (or if deleting, omit) new files
+  - 5) swap top level
+ - ** whole deploy can be atomic(-ish) - do this for everything **
+  - just a '/deploy/current' dir
+  - some things (venvs) cannot be moved, thus the /deploy/venvs dir
+  - ** ensure (enforce) equivalent relpath nesting
+"""
+import os.path
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.os.paths import is_path_in_dir
+from omlish.os.paths import relative_symlink
+
+from .paths import SingleDirDeployPathOwner
+from .specs import AppDeployConfLink
+from .specs import DeployConfFile
+from .specs import DeployConfLink
+from .specs import DeployConfSpec
+from .specs import TagDeployConfLink
+from .types import DeployAppTag
+from .types import DeployHome
+
+
+class DeployConfManager(SingleDirDeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: ta.Optional[DeployHome] = None,
+    ) -> None:
+        super().__init__(
+            owned_dir='conf',
+            deploy_home=deploy_home,
+        )
+
+    async def _write_conf_file(
+            self,
+            cf: DeployConfFile,
+            conf_dir: str,
+    ) -> None:
+        conf_file = os.path.join(conf_dir, cf.path)
+        check.arg(is_path_in_dir(conf_dir, conf_file))
+
+        os.makedirs(os.path.dirname(conf_file), exist_ok=True)
+
+        with open(conf_file, 'w') as f:  # noqa
+            f.write(cf.body)
+
+    async def _make_conf_link(
+            self,
+            link: DeployConfLink,
+            conf_dir: str,
+            app_tag: DeployAppTag,
+            link_dir: str,
+    ) -> None:
+        link_src = os.path.join(conf_dir, link.src)
+        check.arg(is_path_in_dir(conf_dir, link_src))
+
+        is_link_dir = link.src.endswith('/')
+        if is_link_dir:
+            check.arg(link.src.count('/') == 1)
+            check.arg(os.path.isdir(link_src))
+            link_dst_pfx = link.src
+            link_dst_sfx = ''
+
+        elif '/' in link.src:
+            check.arg(os.path.isfile(link_src))
+            d, f = os.path.split(link.src)
+            # TODO: check filename :|
+            link_dst_pfx = d + '/'
+            link_dst_sfx = '-' + f
+
+        else:
+            check.arg(os.path.isfile(link_src))
+            if '.' in link.src:
+                l, _, r = link.src.partition('.')
+                link_dst_pfx = l + '/'
+                link_dst_sfx = '.' + r
+            else:
+                link_dst_pfx = link.src + '/'
+                link_dst_sfx = ''
+
+        if isinstance(link, AppDeployConfLink):
+            link_dst_mid = str(app_tag.app)
+            sym_root = link_dir
+        elif isinstance(link, TagDeployConfLink):
+            link_dst_mid = '-'.join([app_tag.app, app_tag.tag])
+            sym_root = conf_dir
+        else:
+            raise TypeError(link)
+
+        link_dst = ''.join([
+            link_dst_pfx,
+            link_dst_mid,
+            link_dst_sfx,
+        ])
+
+        root_conf_dir = self._make_dir()
+        sym_src = os.path.join(sym_root, link.src)
+        sym_dst = os.path.join(root_conf_dir, link_dst)
+        check.arg(is_path_in_dir(root_conf_dir, sym_dst))
+
+        os.makedirs(os.path.dirname(sym_dst), exist_ok=True)
+        relative_symlink(sym_src, sym_dst, target_is_directory=is_link_dir)
+
+    async def write_conf(
+            self,
+            spec: DeployConfSpec,
+            conf_dir: str,
+            app_tag: DeployAppTag,
+            link_dir: str,
+    ) -> None:
+        conf_dir = os.path.abspath(conf_dir)
+        os.makedirs(conf_dir)
+
+        #
+
+        for cf in spec.files or []:
+            await self._write_conf_file(
+                cf,
+                conf_dir,
+            )
+
+        #
+
+        for link in spec.links or []:
+            await self._make_conf_link(
+                link,
+                conf_dir,
+                app_tag,
+                link_dir,
+            )

ominfra/manage/deploy/git.py

@@ -18,8 +18,8 @@ from omlish.lite.check import check
 from omlish.os.atomics import AtomicPathSwapping
 
 from .paths import SingleDirDeployPathOwner
-from .specs import DeployGitCheckout
 from .specs import DeployGitRepo
+from .specs import DeployGitSpec
 from .types import DeployHome
 from .types import DeployRev
 
@@ -95,7 +95,7 @@ class DeployGitManager(SingleDirDeployPathOwner):
 
         #
 
-        async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
+        async def checkout(self, spec: DeployGitSpec, dst_dir: str) -> None:
             check.state(not os.path.exists(dst_dir))
             with self._git._atomics.begin_atomic_path_swap(  # noqa
                     'dir',
@@ -103,14 +103,14 @@ class DeployGitManager(SingleDirDeployPathOwner):
                     auto_commit=True,
                     make_dirs=True,
             ) as dst_swap:
-                await self.fetch(checkout.rev)
+                await self.fetch(spec.rev)
 
                 dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
                 await dst_call('git', 'init')
 
                 await dst_call('git', 'remote', 'add', 'local', self._dir)
-                await dst_call('git', 'fetch', '--depth=1', 'local', checkout.rev)
-                await dst_call('git', 'checkout', checkout.rev, *(checkout.subtrees or []))
+                await dst_call('git', 'fetch', '--depth=1', 'local', spec.rev)
+                await dst_call('git', 'checkout', spec.rev, *(spec.subtrees or []))
 
     def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
         try:
@@ -119,5 +119,9 @@ class DeployGitManager(SingleDirDeployPathOwner):
             repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
             return repo_dir
 
-    async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
-        await self.get_repo_dir(checkout.repo).checkout(checkout, dst_dir)
+    async def checkout(
+            self,
+            spec: DeployGitSpec,
+            dst_dir: str,
+    ) -> None:
+        await self.get_repo_dir(spec.repo).checkout(spec, dst_dir)

ominfra/manage/deploy/inject.py

@@ -11,10 +11,13 @@ from ..commands.inject import bind_command
 from .apps import DeployAppManager
 from .commands import DeployCommand
 from .commands import DeployCommandExecutor
+from .conf import DeployConfManager
 from .config import DeployConfig
 from .git import DeployGitManager
 from .interp import InterpCommand
 from .interp import InterpCommandExecutor
+from .paths import DeployPathOwner
+from .paths import DeployPathOwners
 from .tmp import DeployTmpManager
 from .types import DeployHome
 from .venvs import DeployVenvManager
@@ -26,23 +29,45 @@ def bind_deploy(
 ) -> InjectorBindings:
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(deploy_config),
+    ]
+
+    #
+
+    def bind_manager(cls: type) -> InjectorBindings:
+        return inj.as_bindings(
+            inj.bind(cls, singleton=True),
+
+            *([inj.bind(DeployPathOwner, to_key=cls, array=True)] if issubclass(cls, DeployPathOwner) else []),
+        )
 
-        #
+    lst.extend([
+        inj.bind_array(DeployPathOwner),
+        inj.bind_array_type(DeployPathOwner, DeployPathOwners),
+    ])
 
-        inj.bind(DeployAppManager, singleton=True),
+    #
 
-        inj.bind(DeployGitManager, singleton=True),
+    lst.extend([
+        bind_manager(DeployAppManager),
 
-        inj.bind(DeployTmpManager, singleton=True),
+        bind_manager(DeployConfManager),
+
+        bind_manager(DeployGitManager),
+
+        bind_manager(DeployTmpManager),
         inj.bind(AtomicPathSwapping, to_key=DeployTmpManager),
 
-        inj.bind(DeployVenvManager, singleton=True),
+        bind_manager(DeployVenvManager),
+    ])
 
-        #
+    #
 
+    lst.extend([
         bind_command(DeployCommand, DeployCommandExecutor),
         bind_command(InterpCommand, InterpCommandExecutor),
-    ]
+    ])
+
+    #
 
     if (dh := deploy_config.deploy_home) is not None:
         dh = os.path.abspath(os.path.expanduser(dh))

ominfra/manage/deploy/paths.py

@@ -1,11 +1,12 @@
 # ruff: noqa: UP006 UP007
 """
 TODO:
- - run/pidfile
+ - run/{.pid,.sock}
  - logs/...
  - current symlink
  - conf/{nginx,supervisor}
  - env/?
+ - apps/<app>/shared
 """
 import abc
 import dataclasses as dc
@@ -208,6 +209,9 @@ class DeployPathOwner(abc.ABC):
         raise NotImplementedError
 
 
+DeployPathOwners = ta.NewType('DeployPathOwners', ta.Sequence[DeployPathOwner])
+
+
 class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
     def __init__(
             self,

ominfra/manage/deploy/specs.py

@@ -1,4 +1,5 @@
 # ruff: noqa: UP006 UP007
+import abc
 import dataclasses as dc
 import hashlib
 import typing as ta
@@ -14,6 +15,17 @@ from .types import DeployRev
 ##
 
 
+def check_valid_deploy_spec_path(s: str) -> str:
+    check.non_empty_str(s)
+    for c in ['..', '//']:
+        check.not_in(c, s)
+    check.arg(not s.startswith('/'))
+    return s
+
+
+##
+
+
 @dc.dataclass(frozen=True)
 class DeployGitRepo:
     host: ta.Optional[str] = None
@@ -26,7 +38,7 @@ class DeployGitRepo:
 
 
 @dc.dataclass(frozen=True)
-class DeployGitCheckout:
+class DeployGitSpec:
     repo: DeployGitRepo
     rev: DeployRev
 
@@ -52,8 +64,62 @@ class DeployVenvSpec:
 
     use_uv: bool = False
 
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployConfFile:
+    path: str
+    body: str
+
     def __post_init__(self) -> None:
-        hash(self)
+        check_valid_deploy_spec_path(self.path)
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class DeployConfLink(abc.ABC):  # noqa
+    """
+    May be either:
+     - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
+     - @conf/file - links a single file in a single subdir to conf/@conf/@dst-file
+     - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
+    """
+
+    src: str
+
+    def __post_init__(self) -> None:
+        check_valid_deploy_spec_path(self.src)
+        if '/' in self.src:
+            check.equal(self.src.count('/'), 1)
+
+
+class AppDeployConfLink(DeployConfLink):
+    pass
+
+
+class TagDeployConfLink(DeployConfLink):
+    pass
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class DeployConfSpec:
+    files: ta.Optional[ta.Sequence[DeployConfFile]] = None
+
+    links: ta.Optional[ta.Sequence[DeployConfLink]] = None
+
+    def __post_init__(self) -> None:
+        if self.files:
+            seen: ta.Set[str] = set()
+            for f in self.files:
+                check.not_in(f.path, seen)
+                seen.add(f.path)
 
 
 ##
@@ -62,12 +128,12 @@ class DeployVenvSpec:
 @dc.dataclass(frozen=True)
 class DeploySpec:
     app: DeployApp
-    checkout: DeployGitCheckout
+
+    git: DeployGitSpec
 
     venv: ta.Optional[DeployVenvSpec] = None
 
-    def __post_init__(self) -> None:
-        hash(self)
+    conf: ta.Optional[DeployConfSpec] = None
 
     @cached_nullary
     def key(self) -> DeployKey:

ominfra/manage/deploy/venvs.py

@@ -5,46 +5,28 @@ TODO:
  - share more code with pyproject?
 """
 import os.path
-import typing as ta
 
 from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
-from omlish.lite.cached import cached_nullary
-from omlish.lite.check import check
 from omlish.os.atomics import AtomicPathSwapping
 
-from .paths import DeployPath
-from .paths import DeployPathOwner
 from .specs import DeployVenvSpec
-from .types import DeployAppTag
-from .types import DeployHome
 
 
-class DeployVenvManager(DeployPathOwner):
+class DeployVenvManager:
     def __init__(
             self,
             *,
-            deploy_home: ta.Optional[DeployHome] = None,
             atomics: AtomicPathSwapping,
     ) -> None:
         super().__init__()
 
-        self._deploy_home = deploy_home
         self._atomics = atomics
 
-    @cached_nullary
-    def _dir(self) -> str:
-        return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
-
-    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
-        return {
-            DeployPath.parse('venvs/@app/@tag/'),
-        }
-
     async def setup_venv(
             self,
-            app_dir: str,
-            venv_dir: str,
             spec: DeployVenvSpec,
+            git_dir: str,
+            venv_dir: str,
     ) -> None:
         sys_exe = 'python3'
 
@@ -58,7 +40,7 @@ class DeployVenvManager(DeployPathOwner):
 
         #
 
-        reqs_txt = os.path.join(app_dir, 'requirements.txt')
+        reqs_txt = os.path.join(git_dir, 'requirements.txt')
 
         if os.path.isfile(reqs_txt):
             if spec.use_uv:
@@ -68,14 +50,3 @@ class DeployVenvManager(DeployPathOwner):
                 pip_cmd = ['-m', 'pip']
 
             await asyncio_subprocesses.check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
-
-    async def setup_app_venv(
-            self,
-            app_tag: DeployAppTag,
-            spec: DeployVenvSpec,
-    ) -> None:
-        await self.setup_venv(
-            os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
-            os.path.join(self._dir(), app_tag.app, app_tag.tag),
-            spec,
-        )

ominfra/scripts/journald2aws.py

@@ -1527,6 +1527,15 @@ def snake_case(name: str) -> str:
 ##
 
 
+def strip_with_newline(s: str) -> str:
+    if not s:
+        return ''
+    return s.strip() + '\n'
+
+
+##
+
+
 def is_dunder(name: str) -> bool:
     return (
         name[:2] == name[-2:] == '__' and

ominfra/scripts/manage.py

@@ -2732,6 +2732,15 @@ def snake_case(name: str) -> str:
 ##
 
 
+def strip_with_newline(s: str) -> str:
+    if not s:
+        return ''
+    return s.strip() + '\n'
+
+
+##
+
+
 def is_dunder(name: str) -> bool:
     return (
         name[:2] == name[-2:] == '__' and
@@ -3392,6 +3401,38 @@ class LinuxOsRelease:
         return dct
 
 
+########################################
+# ../../../omlish/os/paths.py
+
+
+def abs_real_path(p: str) -> str:
+    return os.path.abspath(os.path.realpath(p))
+
+
+def is_path_in_dir(base_dir: str, target_path: str) -> bool:
+    base_dir = abs_real_path(base_dir)
+    target_path = abs_real_path(target_path)
+
+    return target_path.startswith(base_dir + os.path.sep)
+
+
+def relative_symlink(
+        src: str,
+        dst: str,
+        *,
+        target_is_directory: bool = False,
+        dir_fd: ta.Optional[int] = None,
+        **kwargs: ta.Any,
+) -> None:
+    os.symlink(
+        os.path.relpath(src, os.path.dirname(dst)),
+        dst,
+        target_is_directory=target_is_directory,
+        dir_fd=dir_fd,
+        **kwargs,
+    )
+
+
 ########################################
 # ../../../omdev/packaging/specifiers.py
 # Copyright (c) Donald Stufft and individual contributors.
@@ -4068,11 +4109,12 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
 # ../deploy/paths.py
 """
 TODO:
- - run/pidfile
+ - run/{.pid,.sock}
  - logs/...
  - current symlink
  - conf/{nginx,supervisor}
  - env/?
+ - apps/<app>/shared
 """
 
 
@@ -4262,6 +4304,9 @@ class DeployPathOwner(abc.ABC):
         raise NotImplementedError
 
 
+DeployPathOwners = ta.NewType('DeployPathOwners', ta.Sequence[DeployPathOwner])
+
+
 class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
     def __init__(
             self,
@@ -4300,6 +4345,17 @@ class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
 ##
 
 
+def check_valid_deploy_spec_path(s: str) -> str:
+    check.non_empty_str(s)
+    for c in ['..', '//']:
+        check.not_in(c, s)
+    check.arg(not s.startswith('/'))
+    return s
+
+
+##
+
+
 @dc.dataclass(frozen=True)
 class DeployGitRepo:
     host: ta.Optional[str] = None
@@ -4312,7 +4368,7 @@ class DeployGitRepo:
 
 
 @dc.dataclass(frozen=True)
-class DeployGitCheckout:
+class DeployGitSpec:
     repo: DeployGitRepo
     rev: DeployRev
 
@@ -4338,8 +4394,62 @@ class DeployVenvSpec:
 
     use_uv: bool = False
 
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployConfFile:
+    path: str
+    body: str
+
     def __post_init__(self) -> None:
-        hash(self)
+        check_valid_deploy_spec_path(self.path)
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class DeployConfLink(abc.ABC):  # noqa
+    """
+    May be either:
+     - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
+     - @conf/file - links a single file in a single subdir to conf/@conf/@dst-file
+     - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
+    """
+
+    src: str
+
+    def __post_init__(self) -> None:
+        check_valid_deploy_spec_path(self.src)
+        if '/' in self.src:
+            check.equal(self.src.count('/'), 1)
+
+
+class AppDeployConfLink(DeployConfLink):
+    pass
+
+
+class TagDeployConfLink(DeployConfLink):
+    pass
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class DeployConfSpec:
+    files: ta.Optional[ta.Sequence[DeployConfFile]] = None
+
+    links: ta.Optional[ta.Sequence[DeployConfLink]] = None
+
+    def __post_init__(self) -> None:
+        if self.files:
+            seen: ta.Set[str] = set()
+            for f in self.files:
+                check.not_in(f.path, seen)
+                seen.add(f.path)
 
 
 ##
@@ -4348,12 +4458,12 @@ class DeployVenvSpec:
 @dc.dataclass(frozen=True)
 class DeploySpec:
     app: DeployApp
-    checkout: DeployGitCheckout
+
+    git: DeployGitSpec
 
     venv: ta.Optional[DeployVenvSpec] = None
 
-    def __post_init__(self) -> None:
-        hash(self)
+    conf: ta.Optional[DeployConfSpec] = None
 
     @cached_nullary
     def key(self) -> DeployKey:
@@ -6435,7 +6545,7 @@ class AtomicPathSwapping(abc.ABC):
 ##
 
 
-class OsRenameAtomicPathSwap(AtomicPathSwap):
+class OsReplaceAtomicPathSwap(AtomicPathSwap):
     def __init__(
             self,
             kind: AtomicPathSwapKind,
@@ -6463,7 +6573,7 @@ class OsRenameAtomicPathSwap(AtomicPathSwap):
         return self._tmp_path
 
     def _commit(self) -> None:
-        os.rename(self._tmp_path, self._dst_path)
+        os.replace(self._tmp_path, self._dst_path)
 
     def _abort(self) -> None:
         shutil.rmtree(self._tmp_path, ignore_errors=True)
@@ -6510,7 +6620,7 @@ class TempDirAtomicPathSwapping(AtomicPathSwapping):
         else:
             raise TypeError(kind)
 
-        return OsRenameAtomicPathSwap(
+        return OsReplaceAtomicPathSwap(
             kind,
             dst_path,
             tmp_path,
@@ -6748,6 +6858,137 @@ class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
 CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
 
 
+########################################
+# ../deploy/conf.py
+"""
+TODO:
+ - @conf DeployPathPlaceholder? :|
+ - post-deploy: remove any dir_links not present in new spec
+  - * only if succeeded * - otherwise, remove any dir_links present in new spec but not previously present?
+   - no such thing as 'previously present'.. build a 'deploy state' and pass it back?
+ - ** whole thing can be atomic **
+  - 1) new atomic temp dir
+  - 2) for each subdir not needing modification, hardlink into temp dir
+  - 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
+  - 4) write (or if deleting, omit) new files
+  - 5) swap top level
+ - ** whole deploy can be atomic(-ish) - do this for everything **
+  - just a '/deploy/current' dir
+  - some things (venvs) cannot be moved, thus the /deploy/venvs dir
+  - ** ensure (enforce) equivalent relpath nesting
+"""
+
+
+class DeployConfManager(SingleDirDeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: ta.Optional[DeployHome] = None,
+    ) -> None:
+        super().__init__(
+            owned_dir='conf',
+            deploy_home=deploy_home,
+        )
+
+    async def _write_conf_file(
+            self,
+            cf: DeployConfFile,
+            conf_dir: str,
+    ) -> None:
+        conf_file = os.path.join(conf_dir, cf.path)
+        check.arg(is_path_in_dir(conf_dir, conf_file))
+
+        os.makedirs(os.path.dirname(conf_file), exist_ok=True)
+
+        with open(conf_file, 'w') as f:  # noqa
+            f.write(cf.body)
+
+    async def _make_conf_link(
+            self,
+            link: DeployConfLink,
+            conf_dir: str,
+            app_tag: DeployAppTag,
+            link_dir: str,
+    ) -> None:
+        link_src = os.path.join(conf_dir, link.src)
+        check.arg(is_path_in_dir(conf_dir, link_src))
+
+        is_link_dir = link.src.endswith('/')
+        if is_link_dir:
+            check.arg(link.src.count('/') == 1)
+            check.arg(os.path.isdir(link_src))
+            link_dst_pfx = link.src
+            link_dst_sfx = ''
+
+        elif '/' in link.src:
+            check.arg(os.path.isfile(link_src))
+            d, f = os.path.split(link.src)
+            # TODO: check filename :|
+            link_dst_pfx = d + '/'
+            link_dst_sfx = '-' + f
+
+        else:
+            check.arg(os.path.isfile(link_src))
+            if '.' in link.src:
+                l, _, r = link.src.partition('.')
+                link_dst_pfx = l + '/'
+                link_dst_sfx = '.' + r
+            else:
+                link_dst_pfx = link.src + '/'
+                link_dst_sfx = ''
+
+        if isinstance(link, AppDeployConfLink):
+            link_dst_mid = str(app_tag.app)
+            sym_root = link_dir
+        elif isinstance(link, TagDeployConfLink):
+            link_dst_mid = '-'.join([app_tag.app, app_tag.tag])
+            sym_root = conf_dir
+        else:
+            raise TypeError(link)
+
+        link_dst = ''.join([
+            link_dst_pfx,
+            link_dst_mid,
+            link_dst_sfx,
+        ])
+
+        root_conf_dir = self._make_dir()
+        sym_src = os.path.join(sym_root, link.src)
+        sym_dst = os.path.join(root_conf_dir, link_dst)
+        check.arg(is_path_in_dir(root_conf_dir, sym_dst))
+
+        os.makedirs(os.path.dirname(sym_dst), exist_ok=True)
+        relative_symlink(sym_src, sym_dst, target_is_directory=is_link_dir)
+
+    async def write_conf(
+            self,
+            spec: DeployConfSpec,
+            conf_dir: str,
+            app_tag: DeployAppTag,
+            link_dir: str,
+    ) -> None:
+        conf_dir = os.path.abspath(conf_dir)
+        os.makedirs(conf_dir)
+
+        #
+
+        for cf in spec.files or []:
+            await self._write_conf_file(
+                cf,
+                conf_dir,
+            )
+
+        #
+
+        for link in spec.links or []:
+            await self._make_conf_link(
+                link,
+                conf_dir,
+                app_tag,
+                link_dir,
+            )
+
+
 ########################################
 # ../deploy/tmp.py
 
@@ -8240,7 +8481,7 @@ class DeployGitManager(SingleDirDeployPathOwner):
 
         #
 
-        async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
+        async def checkout(self, spec: DeployGitSpec, dst_dir: str) -> None:
             check.state(not os.path.exists(dst_dir))
             with self._git._atomics.begin_atomic_path_swap(  # noqa
                     'dir',
@@ -8248,14 +8489,14 @@ class DeployGitManager(SingleDirDeployPathOwner):
                     auto_commit=True,
                     make_dirs=True,
             ) as dst_swap:
-                await self.fetch(checkout.rev)
+                await self.fetch(spec.rev)
 
                 dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
                 await dst_call('git', 'init')
 
                 await dst_call('git', 'remote', 'add', 'local', self._dir)
-                await dst_call('git', 'fetch', '--depth=1', 'local', checkout.rev)
-                await dst_call('git', 'checkout', checkout.rev, *(checkout.subtrees or []))
+                await dst_call('git', 'fetch', '--depth=1', 'local', spec.rev)
+                await dst_call('git', 'checkout', spec.rev, *(spec.subtrees or []))
 
     def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
         try:
@@ -8264,8 +8505,12 @@ class DeployGitManager(SingleDirDeployPathOwner):
             repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
             return repo_dir
 
-    async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
-        await self.get_repo_dir(checkout.repo).checkout(checkout, dst_dir)
+    async def checkout(
+            self,
+            spec: DeployGitSpec,
+            dst_dir: str,
+    ) -> None:
+        await self.get_repo_dir(spec.repo).checkout(spec, dst_dir)
 
 
 ########################################
@@ -8277,32 +8522,21 @@ TODO:
 """
 
 
-class DeployVenvManager(DeployPathOwner):
+class DeployVenvManager:
     def __init__(
             self,
             *,
-            deploy_home: ta.Optional[DeployHome] = None,
             atomics: AtomicPathSwapping,
     ) -> None:
         super().__init__()
 
-        self._deploy_home = deploy_home
         self._atomics = atomics
 
-    @cached_nullary
-    def _dir(self) -> str:
-        return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
-
-    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
-        return {
-            DeployPath.parse('venvs/@app/@tag/'),
-        }
-
     async def setup_venv(
             self,
-            app_dir: str,
-            venv_dir: str,
             spec: DeployVenvSpec,
+            git_dir: str,
+            venv_dir: str,
     ) -> None:
         sys_exe = 'python3'
 
@@ -8316,7 +8550,7 @@ class DeployVenvManager(DeployPathOwner):
 
         #
 
-        reqs_txt = os.path.join(app_dir, 'requirements.txt')
+        reqs_txt = os.path.join(git_dir, 'requirements.txt')
 
         if os.path.isfile(reqs_txt):
             if spec.use_uv:
@@ -8327,17 +8561,6 @@ class DeployVenvManager(DeployPathOwner):
 
             await asyncio_subprocesses.check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
 
-    async def setup_app_venv(
-            self,
-            app_tag: DeployAppTag,
-            spec: DeployVenvSpec,
-    ) -> None:
-        await self.setup_venv(
-            os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
-            os.path.join(self._dir(), app_tag.app, app_tag.tag),
-            spec,
-        )
-
 
 ########################################
 # ../remote/_main.py
@@ -8886,12 +9109,16 @@ class DeployAppManager(DeployPathOwner):
             self,
             *,
             deploy_home: ta.Optional[DeployHome] = None,
+
+            conf: DeployConfManager,
             git: DeployGitManager,
             venvs: DeployVenvManager,
     ) -> None:
         super().__init__()
 
         self._deploy_home = deploy_home
+
+        self._conf = conf
         self._git = git
         self._venvs = venvs
 
@@ -8901,27 +9128,72 @@ class DeployAppManager(DeployPathOwner):
 
     def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
         return {
-            DeployPath.parse('apps/@app/@tag/'),
+            DeployPath.parse('apps/@app/current'),
+            DeployPath.parse('apps/@app/deploying'),
+
+            DeployPath.parse('apps/@app/tags/@tag/conf/'),
+            DeployPath.parse('apps/@app/tags/@tag/git/'),
+            DeployPath.parse('apps/@app/tags/@tag/venv/'),
         }
 
     async def prepare_app(
             self,
             spec: DeploySpec,
     ) -> None:
-        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.checkout.rev, spec.key()))
-        app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
+        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.git.rev, spec.key()))
+
+        #
+
+        app_dir = os.path.join(self._dir(), spec.app)
+        os.makedirs(app_dir, exist_ok=True)
 
         #
 
+        tag_dir = os.path.join(app_dir, 'tags', app_tag.tag)
+        os.makedirs(tag_dir)
+
+        #
+
+        deploying_file = os.path.join(app_dir, 'deploying')
+        current_file = os.path.join(app_dir, 'current')
+
+        if os.path.exists(deploying_file):
+            os.unlink(deploying_file)
+        relative_symlink(tag_dir, deploying_file, target_is_directory=True)
+
+        #
+
+        git_dir = os.path.join(tag_dir, 'git')
         await self._git.checkout(
-            spec.checkout,
-            app_dir,
+            spec.git,
+            git_dir,
         )
 
         #
 
         if spec.venv is not None:
-            await self._venvs.setup_app_venv(app_tag, spec.venv)
+            venv_dir = os.path.join(tag_dir, 'venv')
+            await self._venvs.setup_venv(
+                spec.venv,
+                git_dir,
+                venv_dir,
+            )
+
+        #
+
+        if spec.conf is not None:
+            conf_dir = os.path.join(tag_dir, 'conf')
+            conf_link_dir = os.path.join(current_file, 'conf')
+            await self._conf.write_conf(
+                spec.conf,
+                conf_dir,
+                app_tag,
+                conf_link_dir,
+            )
+
+        #
+
+        os.replace(deploying_file, current_file)
 
 
 ########################################
@@ -10043,23 +10315,45 @@ def bind_deploy(
 ) -> InjectorBindings:
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(deploy_config),
+    ]
 
-        #
+    #
 
-        inj.bind(DeployAppManager, singleton=True),
+    def bind_manager(cls: type) -> InjectorBindings:
+        return inj.as_bindings(
+            inj.bind(cls, singleton=True),
 
-        inj.bind(DeployGitManager, singleton=True),
+            *([inj.bind(DeployPathOwner, to_key=cls, array=True)] if issubclass(cls, DeployPathOwner) else []),
+        )
+
+    lst.extend([
+        inj.bind_array(DeployPathOwner),
+        inj.bind_array_type(DeployPathOwner, DeployPathOwners),
+    ])
+
+    #
+
+    lst.extend([
+        bind_manager(DeployAppManager),
 
-        inj.bind(DeployTmpManager, singleton=True),
+        bind_manager(DeployConfManager),
+
+        bind_manager(DeployGitManager),
+
+        bind_manager(DeployTmpManager),
         inj.bind(AtomicPathSwapping, to_key=DeployTmpManager),
 
-        inj.bind(DeployVenvManager, singleton=True),
+        bind_manager(DeployVenvManager),
+    ])
 
-        #
+    #
 
+    lst.extend([
         bind_command(DeployCommand, DeployCommandExecutor),
         bind_command(InterpCommand, InterpCommandExecutor),
-    ]
+    ])
+
+    #
 
     if (dh := deploy_config.deploy_home) is not None:
         dh = os.path.abspath(os.path.expanduser(dh))

ominfra/scripts/supervisor.py

@@ -2406,6 +2406,15 @@ def snake_case(name: str) -> str:
 ##
 
 
+def strip_with_newline(s: str) -> str:
+    if not s:
+        return ''
+    return s.strip() + '\n'
+
+
+##
+
+
 def is_dunder(name: str) -> bool:
     return (
         name[:2] == name[-2:] == '__' and
@@ -6110,6 +6119,7 @@ class ServerConfig:
 
     groups: ta.Optional[ta.Sequence[ProcessGroupConfig]] = None
 
+    # TODO: implement - make sure to accept broken symlinks
     group_config_dirs: ta.Optional[ta.Sequence[str]] = None
 
     @classmethod

ominfra/supervisor/configs.py

@@ -245,6 +245,7 @@ class ServerConfig:
 
     groups: ta.Optional[ta.Sequence[ProcessGroupConfig]] = None
 
+    # TODO: implement - make sure to accept broken symlinks
     group_config_dirs: ta.Optional[ta.Sequence[str]] = None
 
     @classmethod

{ominfra-0.0.0.dev166.dist-info → ominfra-0.0.0.dev167.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev166
+Version: 0.0.0.dev167
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev166
-Requires-Dist: omlish==0.0.0.dev166
+Requires-Dist: omdev==0.0.0.dev167
+Requires-Dist: omlish==0.0.0.dev167
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.18; extra == "all"

{ominfra-0.0.0.dev166.dist-info → ominfra-0.0.0.dev167.dist-info}/RECORD

@@ -44,17 +44,18 @@ ominfra/manage/commands/ping.py,sha256=DVZFzL1Z_f-Bq53vxMrL3xOi0iK_nMonJE4KvQf9w
 ominfra/manage/commands/subprocess.py,sha256=yHGMbAI-xKe_9BUs5IZ3Yav8qRE-I9aGnBtTwW15Pnw,2440
 ominfra/manage/commands/types.py,sha256=XFZPeqeIBAaIIQF3pdPbGxLlb-LCrz6WtlDWO2q_vz0,210
 ominfra/manage/deploy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ominfra/manage/deploy/apps.py,sha256=Nv3eXVPxEbSz4wyTI8chJ-C-EEC721vFssvvg0YDRXo,1937
+ominfra/manage/deploy/apps.py,sha256=jLvam6sVcWfRmyyLeJyTxz0IErWTn98rf1-8lIEEYlM,3292
 ominfra/manage/deploy/commands.py,sha256=N9qVntnRgJ_IneI7rEQB2Za0oU7gouPfm-sl2MCwW1E,764
+ominfra/manage/deploy/conf.py,sha256=HxKLWIT95StavJ0uHhMhuvWio1gTYBlDJv6N1_XdxEo,4417
 ominfra/manage/deploy/config.py,sha256=aR6ubMEWqkTI55XtcG1Cczn6YhCVN6eSL8DT5EHQJN0,166
-ominfra/manage/deploy/git.py,sha256=6CGLvGH8uYkuT8gyZHybJb7sgUPTtFgy7grj1YHkI9g,3747
-ominfra/manage/deploy/inject.py,sha256=8wuIgdzkDCHbc69nD1meLjwfCOMdWOIfDT5yijL-du8,1441
+ominfra/manage/deploy/git.py,sha256=BGht0UmY1d76kSGHESWuG8qxNKLuOGrwey5V9M4pfCY,3746
+ominfra/manage/deploy/inject.py,sha256=o-bgWvziUuE5XzsE9_rcPSdWqPO9AAf2SbXamEpir2k,1978
 ominfra/manage/deploy/interp.py,sha256=OKkenH8YKEW_mEDR6X7_ZLxK9a1Ox6KHSwFPTHT6OzA,1029
-ominfra/manage/deploy/paths.py,sha256=tK8zZFWOHDRdTN5AlTe-3MpgZqovhWrljGosQmeEYvo,6839
-ominfra/manage/deploy/specs.py,sha256=qQSMA2ns0dalbizYwOp14i154mCOkjyvvh4_cN0gT3k,1574
+ominfra/manage/deploy/paths.py,sha256=Rq9OsK3MpporH6tfuumzLO51e4Yz_wi6RiXEg82iyLY,6947
+ominfra/manage/deploy/specs.py,sha256=Xv1OQ3yQxOTR39Gj5ACgAn7vpgL-zVuGk1CHk_xmt5o,2883
 ominfra/manage/deploy/tmp.py,sha256=L0pIfQuxQ7_6gC_AAv7eubI37_IPzCVR29hkn1MHL2Q,1230
 ominfra/manage/deploy/types.py,sha256=o95wqvTGNRq8Cxx7VpqeX-9x1tI8k8BpqPFvJZkJYBA,305
-ominfra/manage/deploy/venvs.py,sha256=u8ds1TVyipFVSfm7sgEHGPw4JPG_hXY2j1LnTCLjxqw,2337
+ominfra/manage/deploy/venvs.py,sha256=1co8l3eSxl2WccrF-XOTqeltoMccRH63o69NblV3yok,1366
 ominfra/manage/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ominfra/manage/remote/_main.py,sha256=p5KoiS2WMw6QAqlDl_Zun-JybmCsy8awIfpBMLBjGMY,4356
 ominfra/manage/remote/channel.py,sha256=36xR9Ti9ZA8TUBtxmY0u7_3Lv7E6wzQTxlZl7gLR5GE,2224
@@ -75,13 +76,13 @@ ominfra/manage/targets/connection.py,sha256=5e8h9Miej2DKJxZfLyxpGe8y-Y0V_b_AuUW1
 ominfra/manage/targets/inject.py,sha256=P4597xWM-V3I_gCt2O71OLhYQkkXtuJvkYRsIbhhMcE,1561
 ominfra/manage/targets/targets.py,sha256=CFl8Uirgn3gfowO1Fn-LBK-6qYqEMFJ9snPUl0gCRuM,1753
 ominfra/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ominfra/scripts/journald2aws.py,sha256=EC8tSKW3hztBV_Kr_ykK72AmcvnWivUxcz6Sfg3M_hI,155085
-ominfra/scripts/manage.py,sha256=M8k4XmOrq2R3rjUNU3SAHeqdjvWZptpXvFAl4A0RmGM,294060
-ominfra/scripts/supervisor.py,sha256=uPcw4o8gt8xvQ97jXK-WBAaZj3D81Lq9tqDoKxGvLCU,273791
+ominfra/scripts/journald2aws.py,sha256=Cfxnq9N2PaRLBrGSPq4qi39SOGldKZXT_Ve2_uPWcpY,155191
+ominfra/scripts/manage.py,sha256=Yb_Ok9rB8zMycxED0fjzHQVNmVQ2qc14pASuzpves68,301283
+ominfra/scripts/supervisor.py,sha256=y9o5NyKKb7ugwk3ZEywzlqVFd7QDhsSO2_C8oSqO_28,273957
 ominfra/supervisor/LICENSE.txt,sha256=yvqaMNsDhWxziHa9ien6qCW1SkZv-DQlAg96XjfSee8,1746
 ominfra/supervisor/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 ominfra/supervisor/__main__.py,sha256=I0yFw-C08OOiZ3BF6lF1Oiv789EQXu-_j6whDhQUTEA,66
-ominfra/supervisor/configs.py,sha256=InaLW0T93dbAuzKyc6H8wGIFR2oM1GROSiQX9hZY_ko,13711
+ominfra/supervisor/configs.py,sha256=UxcZ1yaw7CiwiSb7A1AegwcWKFXqtswaKaHlIN8Bt_Q,13771
 ominfra/supervisor/dispatchers.py,sha256=zXLwQS4Vc6dWw5o9QOL04UMDt7w6CKu9wf19CjUiS2Q,1005
 ominfra/supervisor/dispatchersimpl.py,sha256=q3dEyOHWTPKm28nmAGisjgIW1BX6O3-SzbYa7nWuTEs,11349
 ominfra/supervisor/events.py,sha256=XGrtzHr1xm0dwjz329fn9eR0_Ap-LQL6Sk8LJ8eVDEo,6692
@@ -119,9 +120,9 @@ ominfra/tailscale/api.py,sha256=C5-t_b6jZXUWcy5k8bXm7CFnk73pSdrlMOgGDeGVrpw,1370
 ominfra/tailscale/cli.py,sha256=h6akQJMl0KuWLHS7Ur6WcBZ2JwF0DJQhsPTnFBdGyNk,3571
 ominfra/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ominfra/tools/listresources.py,sha256=4qVg5txsb10EHhvqXXeM6gJ2jx9LbroEnPydDv1uXs0,6176
-ominfra-0.0.0.dev166.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-ominfra-0.0.0.dev166.dist-info/METADATA,sha256=K1Yk7mm_LiNl0FtGC69NQqG23xD68-dpAwbNiBgfn90,731
-ominfra-0.0.0.dev166.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ominfra-0.0.0.dev166.dist-info/entry_points.txt,sha256=kgecQ2MgGrM9qK744BoKS3tMesaC3yjLnl9pa5CRczg,37
-ominfra-0.0.0.dev166.dist-info/top_level.txt,sha256=E-b2OHkk_AOBLXHYZQ2EOFKl-_6uOGd8EjeG-Zy6h_w,8
-ominfra-0.0.0.dev166.dist-info/RECORD,,
+ominfra-0.0.0.dev167.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+ominfra-0.0.0.dev167.dist-info/METADATA,sha256=JFB_UaK98rzNZ6vQWBEVI9f7zjJ4DeSf00RpqMQrczs,731
+ominfra-0.0.0.dev167.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ominfra-0.0.0.dev167.dist-info/entry_points.txt,sha256=kgecQ2MgGrM9qK744BoKS3tMesaC3yjLnl9pa5CRczg,37
+ominfra-0.0.0.dev167.dist-info/top_level.txt,sha256=E-b2OHkk_AOBLXHYZQ2EOFKl-_6uOGd8EjeG-Zy6h_w,8
+ominfra-0.0.0.dev167.dist-info/RECORD,,