ominfra 0.0.0.dev165__tar.gz → 0.0.0.dev167__tar.gz

{ominfra-0.0.0.dev165/ominfra.egg-info → ominfra-0.0.0.dev167}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev165
+Version: 0.0.0.dev167
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev165
-Requires-Dist: omlish==0.0.0.dev165
+Requires-Dist: omdev==0.0.0.dev167
+Requires-Dist: omlish==0.0.0.dev167
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.18; extra == "all"

ominfra-0.0.0.dev167/ominfra/manage/deploy/apps.py

@@ -0,0 +1,125 @@
+# ruff: noqa: UP006 UP007
+import datetime
+import os.path
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.os.paths import relative_symlink
+
+from .conf import DeployConfManager
+from .git import DeployGitManager
+from .paths import DeployPath
+from .paths import DeployPathOwner
+from .specs import DeploySpec
+from .types import DeployAppTag
+from .types import DeployHome
+from .types import DeployKey
+from .types import DeployRev
+from .types import DeployTag
+from .venvs import DeployVenvManager
+
+
+def make_deploy_tag(
+        rev: DeployRev,
+        key: DeployKey,
+        *,
+        utcnow: ta.Optional[datetime.datetime] = None,
+) -> DeployTag:
+    if utcnow is None:
+        utcnow = datetime.datetime.now(tz=datetime.timezone.utc)  # noqa
+    now_fmt = '%Y%m%dT%H%M%SZ'
+    now_str = utcnow.strftime(now_fmt)
+    return DeployTag('-'.join([now_str, rev, key]))
+
+
+class DeployAppManager(DeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: ta.Optional[DeployHome] = None,
+
+            conf: DeployConfManager,
+            git: DeployGitManager,
+            venvs: DeployVenvManager,
+    ) -> None:
+        super().__init__()
+
+        self._deploy_home = deploy_home
+
+        self._conf = conf
+        self._git = git
+        self._venvs = venvs
+
+    @cached_nullary
+    def _dir(self) -> str:
+        return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
+
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return {
+            DeployPath.parse('apps/@app/current'),
+            DeployPath.parse('apps/@app/deploying'),
+
+            DeployPath.parse('apps/@app/tags/@tag/conf/'),
+            DeployPath.parse('apps/@app/tags/@tag/git/'),
+            DeployPath.parse('apps/@app/tags/@tag/venv/'),
+        }
+
+    async def prepare_app(
+            self,
+            spec: DeploySpec,
+    ) -> None:
+        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.git.rev, spec.key()))
+
+        #
+
+        app_dir = os.path.join(self._dir(), spec.app)
+        os.makedirs(app_dir, exist_ok=True)
+
+        #
+
+        tag_dir = os.path.join(app_dir, 'tags', app_tag.tag)
+        os.makedirs(tag_dir)
+
+        #
+
+        deploying_file = os.path.join(app_dir, 'deploying')
+        current_file = os.path.join(app_dir, 'current')
+
+        if os.path.exists(deploying_file):
+            os.unlink(deploying_file)
+        relative_symlink(tag_dir, deploying_file, target_is_directory=True)
+
+        #
+
+        git_dir = os.path.join(tag_dir, 'git')
+        await self._git.checkout(
+            spec.git,
+            git_dir,
+        )
+
+        #
+
+        if spec.venv is not None:
+            venv_dir = os.path.join(tag_dir, 'venv')
+            await self._venvs.setup_venv(
+                spec.venv,
+                git_dir,
+                venv_dir,
+            )
+
+        #
+
+        if spec.conf is not None:
+            conf_dir = os.path.join(tag_dir, 'conf')
+            conf_link_dir = os.path.join(current_file, 'conf')
+            await self._conf.write_conf(
+                spec.conf,
+                conf_dir,
+                app_tag,
+                conf_link_dir,
+            )
+
+        #
+
+        os.replace(deploying_file, current_file)

ominfra-0.0.0.dev167/ominfra/manage/deploy/conf.py

@@ -0,0 +1,143 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - @conf DeployPathPlaceholder? :|
+ - post-deploy: remove any dir_links not present in new spec
+  - * only if succeeded * - otherwise, remove any dir_links present in new spec but not previously present?
+   - no such thing as 'previously present'.. build a 'deploy state' and pass it back?
+ - ** whole thing can be atomic **
+  - 1) new atomic temp dir
+  - 2) for each subdir not needing modification, hardlink into temp dir
+  - 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
+  - 4) write (or if deleting, omit) new files
+  - 5) swap top level
+ - ** whole deploy can be atomic(-ish) - do this for everything **
+  - just a '/deploy/current' dir
+  - some things (venvs) cannot be moved, thus the /deploy/venvs dir
+  - ** ensure (enforce) equivalent relpath nesting
+"""
+import os.path
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.os.paths import is_path_in_dir
+from omlish.os.paths import relative_symlink
+
+from .paths import SingleDirDeployPathOwner
+from .specs import AppDeployConfLink
+from .specs import DeployConfFile
+from .specs import DeployConfLink
+from .specs import DeployConfSpec
+from .specs import TagDeployConfLink
+from .types import DeployAppTag
+from .types import DeployHome
+
+
+class DeployConfManager(SingleDirDeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: ta.Optional[DeployHome] = None,
+    ) -> None:
+        super().__init__(
+            owned_dir='conf',
+            deploy_home=deploy_home,
+        )
+
+    async def _write_conf_file(
+            self,
+            cf: DeployConfFile,
+            conf_dir: str,
+    ) -> None:
+        conf_file = os.path.join(conf_dir, cf.path)
+        check.arg(is_path_in_dir(conf_dir, conf_file))
+
+        os.makedirs(os.path.dirname(conf_file), exist_ok=True)
+
+        with open(conf_file, 'w') as f:  # noqa
+            f.write(cf.body)
+
+    async def _make_conf_link(
+            self,
+            link: DeployConfLink,
+            conf_dir: str,
+            app_tag: DeployAppTag,
+            link_dir: str,
+    ) -> None:
+        link_src = os.path.join(conf_dir, link.src)
+        check.arg(is_path_in_dir(conf_dir, link_src))
+
+        is_link_dir = link.src.endswith('/')
+        if is_link_dir:
+            check.arg(link.src.count('/') == 1)
+            check.arg(os.path.isdir(link_src))
+            link_dst_pfx = link.src
+            link_dst_sfx = ''
+
+        elif '/' in link.src:
+            check.arg(os.path.isfile(link_src))
+            d, f = os.path.split(link.src)
+            # TODO: check filename :|
+            link_dst_pfx = d + '/'
+            link_dst_sfx = '-' + f
+
+        else:
+            check.arg(os.path.isfile(link_src))
+            if '.' in link.src:
+                l, _, r = link.src.partition('.')
+                link_dst_pfx = l + '/'
+                link_dst_sfx = '.' + r
+            else:
+                link_dst_pfx = link.src + '/'
+                link_dst_sfx = ''
+
+        if isinstance(link, AppDeployConfLink):
+            link_dst_mid = str(app_tag.app)
+            sym_root = link_dir
+        elif isinstance(link, TagDeployConfLink):
+            link_dst_mid = '-'.join([app_tag.app, app_tag.tag])
+            sym_root = conf_dir
+        else:
+            raise TypeError(link)
+
+        link_dst = ''.join([
+            link_dst_pfx,
+            link_dst_mid,
+            link_dst_sfx,
+        ])
+
+        root_conf_dir = self._make_dir()
+        sym_src = os.path.join(sym_root, link.src)
+        sym_dst = os.path.join(root_conf_dir, link_dst)
+        check.arg(is_path_in_dir(root_conf_dir, sym_dst))
+
+        os.makedirs(os.path.dirname(sym_dst), exist_ok=True)
+        relative_symlink(sym_src, sym_dst, target_is_directory=is_link_dir)
+
+    async def write_conf(
+            self,
+            spec: DeployConfSpec,
+            conf_dir: str,
+            app_tag: DeployAppTag,
+            link_dir: str,
+    ) -> None:
+        conf_dir = os.path.abspath(conf_dir)
+        os.makedirs(conf_dir)
+
+        #
+
+        for cf in spec.files or []:
+            await self._write_conf_file(
+                cf,
+                conf_dir,
+            )
+
+        #
+
+        for link in spec.links or []:
+            await self._make_conf_link(
+                link,
+                conf_dir,
+                app_tag,
+                link_dir,
+            )

{ominfra-0.0.0.dev165 → ominfra-0.0.0.dev167}/ominfra/manage/deploy/git.py

@@ -18,8 +18,8 @@ from omlish.lite.check import check
 from omlish.os.atomics import AtomicPathSwapping
 
 from .paths import SingleDirDeployPathOwner
-from .specs import DeployGitCheckout
 from .specs import DeployGitRepo
+from .specs import DeployGitSpec
 from .types import DeployHome
 from .types import DeployRev
 
@@ -95,7 +95,7 @@ class DeployGitManager(SingleDirDeployPathOwner):
 
         #
 
-        async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
+        async def checkout(self, spec: DeployGitSpec, dst_dir: str) -> None:
             check.state(not os.path.exists(dst_dir))
             with self._git._atomics.begin_atomic_path_swap(  # noqa
                     'dir',
@@ -103,14 +103,14 @@ class DeployGitManager(SingleDirDeployPathOwner):
                     auto_commit=True,
                     make_dirs=True,
             ) as dst_swap:
-                await self.fetch(checkout.rev)
+                await self.fetch(spec.rev)
 
                 dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
                 await dst_call('git', 'init')
 
                 await dst_call('git', 'remote', 'add', 'local', self._dir)
-                await dst_call('git', 'fetch', '--depth=1', 'local', checkout.rev)
-                await dst_call('git', 'checkout', checkout.rev, *(checkout.subtrees or []))
+                await dst_call('git', 'fetch', '--depth=1', 'local', spec.rev)
+                await dst_call('git', 'checkout', spec.rev, *(spec.subtrees or []))
 
     def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
         try:
@@ -119,5 +119,9 @@ class DeployGitManager(SingleDirDeployPathOwner):
             repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
             return repo_dir
 
-    async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
-        await self.get_repo_dir(checkout.repo).checkout(checkout, dst_dir)
+    async def checkout(
+            self,
+            spec: DeployGitSpec,
+            dst_dir: str,
+    ) -> None:
+        await self.get_repo_dir(spec.repo).checkout(spec, dst_dir)

{ominfra-0.0.0.dev165 → ominfra-0.0.0.dev167}/ominfra/manage/deploy/inject.py

@@ -11,10 +11,13 @@ from ..commands.inject import bind_command
 from .apps import DeployAppManager
 from .commands import DeployCommand
 from .commands import DeployCommandExecutor
+from .conf import DeployConfManager
 from .config import DeployConfig
 from .git import DeployGitManager
 from .interp import InterpCommand
 from .interp import InterpCommandExecutor
+from .paths import DeployPathOwner
+from .paths import DeployPathOwners
 from .tmp import DeployTmpManager
 from .types import DeployHome
 from .venvs import DeployVenvManager
@@ -26,23 +29,45 @@ def bind_deploy(
 ) -> InjectorBindings:
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(deploy_config),
+    ]
+
+    #
+
+    def bind_manager(cls: type) -> InjectorBindings:
+        return inj.as_bindings(
+            inj.bind(cls, singleton=True),
+
+            *([inj.bind(DeployPathOwner, to_key=cls, array=True)] if issubclass(cls, DeployPathOwner) else []),
+        )
 
-        #
+    lst.extend([
+        inj.bind_array(DeployPathOwner),
+        inj.bind_array_type(DeployPathOwner, DeployPathOwners),
+    ])
 
-        inj.bind(DeployAppManager, singleton=True),
+    #
 
-        inj.bind(DeployGitManager, singleton=True),
+    lst.extend([
+        bind_manager(DeployAppManager),
 
-        inj.bind(DeployTmpManager, singleton=True),
+        bind_manager(DeployConfManager),
+
+        bind_manager(DeployGitManager),
+
+        bind_manager(DeployTmpManager),
         inj.bind(AtomicPathSwapping, to_key=DeployTmpManager),
 
-        inj.bind(DeployVenvManager, singleton=True),
+        bind_manager(DeployVenvManager),
+    ])
 
-        #
+    #
 
+    lst.extend([
         bind_command(DeployCommand, DeployCommandExecutor),
         bind_command(InterpCommand, InterpCommandExecutor),
-    ]
+    ])
+
+    #
 
     if (dh := deploy_config.deploy_home) is not None:
         dh = os.path.abspath(os.path.expanduser(dh))

{ominfra-0.0.0.dev165 → ominfra-0.0.0.dev167}/ominfra/manage/deploy/paths.py

@@ -1,11 +1,12 @@
 # ruff: noqa: UP006 UP007
 """
 TODO:
- - run/pidfile
+ - run/{.pid,.sock}
  - logs/...
  - current symlink
  - conf/{nginx,supervisor}
  - env/?
+ - apps/<app>/shared
 """
 import abc
 import dataclasses as dc
@@ -208,6 +209,9 @@ class DeployPathOwner(abc.ABC):
         raise NotImplementedError
 
 
+DeployPathOwners = ta.NewType('DeployPathOwners', ta.Sequence[DeployPathOwner])
+
+
 class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
     def __init__(
             self,

{ominfra-0.0.0.dev165 → ominfra-0.0.0.dev167}/ominfra/manage/deploy/specs.py

@@ -1,4 +1,5 @@
 # ruff: noqa: UP006 UP007
+import abc
 import dataclasses as dc
 import hashlib
 import typing as ta
@@ -14,6 +15,17 @@ from .types import DeployRev
 ##
 
 
+def check_valid_deploy_spec_path(s: str) -> str:
+    check.non_empty_str(s)
+    for c in ['..', '//']:
+        check.not_in(c, s)
+    check.arg(not s.startswith('/'))
+    return s
+
+
+##
+
+
 @dc.dataclass(frozen=True)
 class DeployGitRepo:
     host: ta.Optional[str] = None
@@ -26,7 +38,7 @@ class DeployGitRepo:
 
 
 @dc.dataclass(frozen=True)
-class DeployGitCheckout:
+class DeployGitSpec:
     repo: DeployGitRepo
     rev: DeployRev
 
@@ -52,8 +64,62 @@ class DeployVenvSpec:
 
     use_uv: bool = False
 
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployConfFile:
+    path: str
+    body: str
+
     def __post_init__(self) -> None:
-        hash(self)
+        check_valid_deploy_spec_path(self.path)
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class DeployConfLink(abc.ABC):  # noqa
+    """
+    May be either:
+     - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
+     - @conf/file - links a single file in a single subdir to conf/@conf/@dst-file
+     - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
+    """
+
+    src: str
+
+    def __post_init__(self) -> None:
+        check_valid_deploy_spec_path(self.src)
+        if '/' in self.src:
+            check.equal(self.src.count('/'), 1)
+
+
+class AppDeployConfLink(DeployConfLink):
+    pass
+
+
+class TagDeployConfLink(DeployConfLink):
+    pass
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class DeployConfSpec:
+    files: ta.Optional[ta.Sequence[DeployConfFile]] = None
+
+    links: ta.Optional[ta.Sequence[DeployConfLink]] = None
+
+    def __post_init__(self) -> None:
+        if self.files:
+            seen: ta.Set[str] = set()
+            for f in self.files:
+                check.not_in(f.path, seen)
+                seen.add(f.path)
 
 
 ##
@@ -62,12 +128,12 @@ class DeployVenvSpec:
 @dc.dataclass(frozen=True)
 class DeploySpec:
     app: DeployApp
-    checkout: DeployGitCheckout
+
+    git: DeployGitSpec
 
     venv: ta.Optional[DeployVenvSpec] = None
 
-    def __post_init__(self) -> None:
-        hash(self)
+    conf: ta.Optional[DeployConfSpec] = None
 
     @cached_nullary
     def key(self) -> DeployKey:

{ominfra-0.0.0.dev165 → ominfra-0.0.0.dev167}/ominfra/manage/deploy/venvs.py

@@ -5,46 +5,28 @@ TODO:
  - share more code with pyproject?
 """
 import os.path
-import typing as ta
 
 from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
-from omlish.lite.cached import cached_nullary
-from omlish.lite.check import check
 from omlish.os.atomics import AtomicPathSwapping
 
-from .paths import DeployPath
-from .paths import DeployPathOwner
 from .specs import DeployVenvSpec
-from .types import DeployAppTag
-from .types import DeployHome
 
 
-class DeployVenvManager(DeployPathOwner):
+class DeployVenvManager:
     def __init__(
             self,
             *,
-            deploy_home: ta.Optional[DeployHome] = None,
             atomics: AtomicPathSwapping,
     ) -> None:
         super().__init__()
 
-        self._deploy_home = deploy_home
         self._atomics = atomics
 
-    @cached_nullary
-    def _dir(self) -> str:
-        return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
-
-    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
-        return {
-            DeployPath.parse('venvs/@app/@tag/'),
-        }
-
     async def setup_venv(
             self,
-            app_dir: str,
-            venv_dir: str,
             spec: DeployVenvSpec,
+            git_dir: str,
+            venv_dir: str,
     ) -> None:
         sys_exe = 'python3'
 
@@ -58,7 +40,7 @@ class DeployVenvManager(DeployPathOwner):
 
         #
 
-        reqs_txt = os.path.join(app_dir, 'requirements.txt')
+        reqs_txt = os.path.join(git_dir, 'requirements.txt')
 
         if os.path.isfile(reqs_txt):
             if spec.use_uv:
@@ -68,14 +50,3 @@ class DeployVenvManager(DeployPathOwner):
                 pip_cmd = ['-m', 'pip']
 
             await asyncio_subprocesses.check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
-
-    async def setup_app_venv(
-            self,
-            app_tag: DeployAppTag,
-            spec: DeployVenvSpec,
-    ) -> None:
-        await self.setup_venv(
-            os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
-            os.path.join(self._dir(), app_tag.app, app_tag.tag),
-            spec,
-        )

{ominfra-0.0.0.dev165 → ominfra-0.0.0.dev167}/ominfra/scripts/journald2aws.py

@@ -1527,6 +1527,15 @@ def snake_case(name: str) -> str:
 ##
 
 
+def strip_with_newline(s: str) -> str:
+    if not s:
+        return ''
+    return s.strip() + '\n'
+
+
+##
+
+
 def is_dunder(name: str) -> bool:
     return (
         name[:2] == name[-2:] == '__' and