ominfra 0.0.0.dev190__tar.gz → 0.0.0.dev192__tar.gz

{ominfra-0.0.0.dev190/ominfra.egg-info → ominfra-0.0.0.dev192}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev190
+Version: 0.0.0.dev192
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,9 +12,9 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev190
-Requires-Dist: omlish==0.0.0.dev190
-Requires-Dist: omserv==0.0.0.dev190
+Requires-Dist: omdev==0.0.0.dev192
+Requires-Dist: omlish==0.0.0.dev192
+Requires-Dist: omserv==0.0.0.dev192
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.18; extra == "all"

{ominfra-0.0.0.dev190 → ominfra-0.0.0.dev192}/ominfra/manage/deploy/apps.py

@@ -1,5 +1,6 @@
 # ruff: noqa: UP006 UP007
 import dataclasses as dc
+import json
 import os.path
 import typing as ta
 
@@ -13,6 +14,7 @@ from .git import DeployGitManager
 from .paths.owners import DeployPathOwner
 from .paths.paths import DeployPath
 from .specs import DeployAppSpec
+from .tags import DeployAppRev
 from .tags import DeployTagMap
 from .types import DeployHome
 from .venvs import DeployVenvManager
@@ -57,9 +59,21 @@ class DeployAppManager(DeployPathOwner):
 
     #
 
+    def _make_tags(self, spec: DeployAppSpec) -> DeployTagMap:
+        return DeployTagMap(
+            spec.app,
+            spec.key(),
+            DeployAppRev(spec.git.rev),
+        )
+
+    #
+
     @dc.dataclass(frozen=True)
     class PreparedApp:
-        app_dir: str
+        spec: DeployAppSpec
+        tags: DeployTagMap
+
+        dir: str
 
         git_dir: ta.Optional[str] = None
         venv_dir: ta.Optional[str] = None
@@ -70,21 +84,30 @@ class DeployAppManager(DeployPathOwner):
             spec: DeployAppSpec,
             home: DeployHome,
             tags: DeployTagMap,
+            *,
+            conf_string_ns: ta.Optional[ta.Mapping[str, ta.Any]] = None,
     ) -> PreparedApp:
         spec_json = json_dumps_pretty(self._msh.marshal_obj(spec))
 
         #
 
+        app_tags = tags.add(*self._make_tags(spec))
+
+        #
+
         check.non_empty_str(home)
 
-        app_dir = os.path.join(home, self.APP_DIR.render(tags))
+        app_dir = os.path.join(home, self.APP_DIR.render(app_tags))
 
         os.makedirs(app_dir, exist_ok=True)
 
         #
 
         rkw: ta.Dict[str, ta.Any] = dict(
-            app_dir=app_dir,
+            spec=spec,
+            tags=app_tags,
+
+            dir=app_dir,
         )
 
         #
@@ -119,11 +142,67 @@ class DeployAppManager(DeployPathOwner):
         if spec.conf is not None:
             conf_dir = os.path.join(app_dir, 'conf')
             rkw.update(conf_dir=conf_dir)
+
+            conf_ns: ta.Dict[str, ta.Any] = dict(
+                **(conf_string_ns or {}),
+                app=spec.app.s,
+                app_dir=app_dir.rstrip('/'),
+            )
+
             await self._conf.write_app_conf(
                 spec.conf,
                 conf_dir,
+                string_ns=conf_ns,
             )
 
         #
 
         return DeployAppManager.PreparedApp(**rkw)
+
+    async def prepare_app_link(
+            self,
+            tags: DeployTagMap,
+            app_dir: str,
+    ) -> PreparedApp:
+        spec_file = os.path.join(app_dir, 'spec.json')
+        with open(spec_file) as f:  # noqa
+            spec_json = f.read()
+
+        spec: DeployAppSpec = self._msh.unmarshal_obj(json.loads(spec_json), DeployAppSpec)
+
+        #
+
+        app_tags = tags.add(*self._make_tags(spec))
+
+        #
+
+        rkw: ta.Dict[str, ta.Any] = dict(
+            spec=spec,
+            tags=app_tags,
+
+            dir=app_dir,
+        )
+
+        #
+
+        git_dir = os.path.join(app_dir, 'git')
+        check.state(os.path.isdir(git_dir))
+        rkw.update(git_dir=git_dir)
+
+        #
+
+        if spec.venv is not None:
+            venv_dir = os.path.join(app_dir, 'venv')
+            check.state(os.path.isdir(venv_dir))
+            rkw.update(venv_dir=venv_dir)
+
+        #
+
+        if spec.conf is not None:
+            conf_dir = os.path.join(app_dir, 'conf')
+            check.state(os.path.isdir(conf_dir))
+            rkw.update(conf_dir=conf_dir)
+
+        #
+
+        return DeployAppManager.PreparedApp(**rkw)

{ominfra-0.0.0.dev190 → ominfra-0.0.0.dev192}/ominfra/manage/deploy/conf/manager.py

@@ -16,6 +16,8 @@ TODO:
   - some things (venvs) cannot be moved, thus the /deploy/venvs dir
   - ** ensure (enforce) equivalent relpath nesting
 """
+import collections.abc
+import functools
 import os.path
 import typing as ta
 
@@ -43,20 +45,66 @@ from .specs import NginxDeployAppConfContent
 from .specs import RawDeployAppConfContent
 
 
+T = ta.TypeVar('T')
+
+
+##
+
+
 class DeployConfManager:
-    def _render_app_conf_content(self, ac: DeployAppConfContent) -> str:
+    def _process_conf_content(
+            self,
+            content: T,
+            *,
+            str_processor: ta.Optional[ta.Callable[[str], str]] = None,
+    ) -> T:
+        def rec(o):
+            if isinstance(o, str):
+                if str_processor is not None:
+                    return type(o)(str_processor(o))
+
+            elif isinstance(o, collections.abc.Mapping):
+                return type(o)([  # type: ignore
+                    (rec(k), rec(v))
+                    for k, v in o.items()
+                ])
+
+            elif isinstance(o, collections.abc.Iterable):
+                return type(o)([  # type: ignore
+                    rec(e) for e in o
+                ])
+
+            return o
+
+        return rec(content)
+
+    #
+
+    def _render_app_conf_content(
+            self,
+            ac: DeployAppConfContent,
+            *,
+            str_processor: ta.Optional[ta.Callable[[str], str]] = None,
+    ) -> str:
+        pcc = functools.partial(
+            self._process_conf_content,
+            str_processor=str_processor,
+        )
+
         if isinstance(ac, RawDeployAppConfContent):
-            return ac.body
+            return pcc(ac.body)
 
         elif isinstance(ac, JsonDeployAppConfContent):
-            return strip_with_newline(json_dumps_pretty(ac.obj))
+            json_obj = pcc(ac.obj)
+            return strip_with_newline(json_dumps_pretty(json_obj))
 
         elif isinstance(ac, IniDeployAppConfContent):
-            return strip_with_newline(render_ini_config(ac.sections))
+            ini_sections = pcc(ac.sections)
+            return strip_with_newline(render_ini_config(ini_sections))
 
         elif isinstance(ac, NginxDeployAppConfContent):
-            ni = NginxConfigItems.of(ac.items)
-            return strip_with_newline(render_nginx_config_str(ni))
+            nginx_items = NginxConfigItems.of(pcc(ac.items))
+            return strip_with_newline(render_nginx_config_str(nginx_items))
 
         else:
             raise TypeError(ac)
@@ -65,11 +113,16 @@ class DeployConfManager:
             self,
             acf: DeployAppConfFile,
             app_conf_dir: str,
+            *,
+            str_processor: ta.Optional[ta.Callable[[str], str]] = None,
     ) -> None:
         conf_file = os.path.join(app_conf_dir, acf.path)
         check.arg(is_path_in_dir(app_conf_dir, conf_file))
 
-        body = self._render_app_conf_content(acf.content)
+        body = self._render_app_conf_content(
+            acf.content,
+            str_processor=str_processor,
+        )
 
         os.makedirs(os.path.dirname(conf_file), exist_ok=True)
 
@@ -80,11 +133,21 @@ class DeployConfManager:
             self,
             spec: DeployAppConfSpec,
             app_conf_dir: str,
+            *,
+            string_ns: ta.Optional[ta.Mapping[str, ta.Any]] = None,
     ) -> None:
+        process_str: ta.Any
+        if string_ns is not None:
+            def process_str(s: str) -> str:
+                return s.format(**string_ns)
+        else:
+            process_str = None
+
         for acf in spec.files or []:
             await self._write_app_conf_file(
                 acf,
                 app_conf_dir,
+                str_processor=process_str,
             )
 
     #

{ominfra-0.0.0.dev190 → ominfra-0.0.0.dev192}/ominfra/manage/deploy/deploy.py

@@ -9,6 +9,7 @@ from omlish.lite.json import json_dumps_pretty
 from omlish.lite.marshal import ObjMarshalerManager
 from omlish.lite.typing import Func0
 from omlish.lite.typing import Func1
+from omlish.os.paths import abs_real_path
 from omlish.os.paths import relative_symlink
 
 from .apps import DeployAppManager
@@ -19,16 +20,17 @@ from .paths.paths import DeployPath
 from .specs import DeployAppSpec
 from .specs import DeploySpec
 from .systemd import DeploySystemdManager
-from .tags import DeployAppRev
+from .tags import DeployApp
 from .tags import DeployTagMap
 from .tags import DeployTime
+from .tmp import DeployHomeAtomics
 from .types import DeployHome
 
 
 ##
 
 
-DEPLOY_TAG_DATETIME_FMT = '%Y%m%dT%H%M%SZ'
+DEPLOY_TAG_DATETIME_FMT = '%Y-%m-%d-T-%H-%M-%S-%f-Z'
 
 
 DeployManagerUtcClock = ta.NewType('DeployManagerUtcClock', Func0[datetime.datetime])
@@ -38,16 +40,24 @@ class DeployManager(DeployPathOwner):
     def __init__(
             self,
             *,
+            atomics: DeployHomeAtomics,
+
             utc_clock: ta.Optional[DeployManagerUtcClock] = None,
     ):
         super().__init__()
 
+        self._atomics = atomics
+
         self._utc_clock = utc_clock
 
     #
 
+    # Home current link just points to CURRENT_DEPLOY_LINK, and is intended for user convenience.
+    HOME_CURRENT_LINK = DeployPath.parse('current')
+
     DEPLOYS_DIR = DeployPath.parse('deploys/')
 
+    # Authoritative current symlink is not in deploy-home, just to prevent accidental corruption.
     CURRENT_DEPLOY_LINK = DeployPath.parse(f'{DEPLOYS_DIR}current')
     DEPLOYING_DEPLOY_LINK = DeployPath.parse(f'{DEPLOYS_DIR}deploying')
 
@@ -80,6 +90,11 @@ class DeployManager(DeployPathOwner):
 
     #
 
+    def render_path(self, home: DeployHome, pth: DeployPath, tags: ta.Optional[DeployTagMap] = None) -> str:
+        return os.path.join(check.non_empty_str(home), pth.render(tags))
+
+    #
+
     def _utc_now(self) -> datetime.datetime:
         if self._utc_clock is not None:
             return self._utc_clock()  # noqa
@@ -89,6 +104,22 @@ class DeployManager(DeployPathOwner):
     def make_deploy_time(self) -> DeployTime:
         return DeployTime(self._utc_now().strftime(DEPLOY_TAG_DATETIME_FMT))
 
+    #
+
+    def make_home_current_link(self, home: DeployHome) -> None:
+        home_current_link = os.path.join(check.non_empty_str(home), self.HOME_CURRENT_LINK.render())
+        current_deploy_link = os.path.join(check.non_empty_str(home), self.CURRENT_DEPLOY_LINK.render())
+        with self._atomics(home).begin_atomic_path_swap(  # noqa
+                'file',
+                home_current_link,
+                auto_commit=True,
+        ) as dst_swap:
+            os.unlink(dst_swap.tmp_path)
+            os.symlink(
+                os.path.relpath(current_deploy_link, os.path.dirname(dst_swap.dst_path)),
+                dst_swap.tmp_path,
+            )
+
 
 ##
 
@@ -130,18 +161,18 @@ class DeployDriver:
     #
 
     @property
-    def deploy_tags(self) -> DeployTagMap:
+    def tags(self) -> DeployTagMap:
         return DeployTagMap(
             self._time,
             self._spec.key(),
         )
 
-    def render_deploy_path(self, pth: DeployPath, tags: ta.Optional[DeployTagMap] = None) -> str:
-        return os.path.join(self._home, pth.render(tags if tags is not None else self.deploy_tags))
+    def render_path(self, pth: DeployPath, tags: ta.Optional[DeployTagMap] = None) -> str:
+        return os.path.join(self._home, pth.render(tags if tags is not None else self.tags))
 
     @property
-    def deploy_dir(self) -> str:
-        return self.render_deploy_path(self._deploys.DEPLOY_DIR)
+    def dir(self) -> str:
+        return self.render_path(self._deploys.DEPLOY_DIR)
 
     #
 
@@ -150,25 +181,37 @@ class DeployDriver:
 
         #
 
+        das: ta.Set[DeployApp] = {a.app for a in self._spec.apps}
+        las: ta.Set[DeployApp] = set(self._spec.app_links.apps)
+        ras: ta.Set[DeployApp] = set(self._spec.app_links.removed_apps)
+        check.empty(das & (las | ras))
+        check.empty(las & ras)
+
+        #
+
         self._paths.validate_deploy_paths()
 
         #
 
-        os.makedirs(self.deploy_dir)
+        os.makedirs(self.dir)
 
         #
 
-        spec_file = self.render_deploy_path(self._deploys.DEPLOY_SPEC_FILE)
+        spec_file = self.render_path(self._deploys.DEPLOY_SPEC_FILE)
         with open(spec_file, 'w') as f:  # noqa
             f.write(spec_json)
 
         #
 
-        deploying_link = self.render_deploy_path(self._deploys.DEPLOYING_DEPLOY_LINK)
+        deploying_link = self.render_path(self._deploys.DEPLOYING_DEPLOY_LINK)
+        current_link = self.render_path(self._deploys.CURRENT_DEPLOY_LINK)
+
+        #
+
         if os.path.exists(deploying_link):
             os.unlink(deploying_link)
         relative_symlink(
-            self.deploy_dir,
+            self.dir,
             deploying_link,
             target_is_directory=True,
             make_dirs=True,
@@ -180,16 +223,30 @@ class DeployDriver:
             self._deploys.APPS_DEPLOY_DIR,
             self._deploys.CONFS_DEPLOY_DIR,
         ]:
-            os.makedirs(self.render_deploy_path(md))
+            os.makedirs(self.render_path(md))
 
         #
 
+        if not self._spec.app_links.exclude_unspecified:
+            cad = abs_real_path(os.path.join(current_link, 'apps'))
+            if os.path.exists(cad):
+                for d in os.listdir(cad):
+                    if (da := DeployApp(d)) not in das and da not in ras:
+                        las.add(da)
+
+        for la in las:
+            await self._drive_app_link(
+                la,
+                current_link,
+            )
+
         for app in self._spec.apps:
-            await self.drive_app_deploy(app)
+            await self._drive_app_deploy(
+                app,
+            )
 
         #
 
-        current_link = self.render_deploy_path(self._deploys.CURRENT_DEPLOY_LINK)
         os.replace(deploying_link, current_link)
 
         #
@@ -197,31 +254,33 @@ class DeployDriver:
         await self._systemd.sync_systemd(
             self._spec.systemd,
             self._home,
-            os.path.join(self.deploy_dir, 'conf', 'systemd'),  # FIXME
+            os.path.join(self.dir, 'conf', 'systemd'),  # FIXME
         )
 
-    #
+        #
 
-    async def drive_app_deploy(self, app: DeployAppSpec) -> None:
-        app_tags = self.deploy_tags.add(
-            app.app,
-            app.key(),
-            DeployAppRev(app.git.rev),
-        )
+        self._deploys.make_home_current_link(self._home)
 
-        #
+    #
+
+    async def _drive_app_deploy(self, app: DeployAppSpec) -> None:
+        current_deploy_link = os.path.join(self._home, self._deploys.CURRENT_DEPLOY_LINK.render())
 
-        da = await self._apps.prepare_app(
+        pa = await self._apps.prepare_app(
             app,
             self._home,
-            app_tags,
+            self.tags,
+            conf_string_ns=dict(
+                deploy_home=self._home,
+                current_deploy_link=current_deploy_link,
+            ),
         )
 
         #
 
-        app_link = self.render_deploy_path(self._deploys.APP_DEPLOY_LINK, app_tags)
+        app_link = self.render_path(self._deploys.APP_DEPLOY_LINK, pa.tags)
         relative_symlink(
-            da.app_dir,
+            pa.dir,
             app_link,
             target_is_directory=True,
             make_dirs=True,
@@ -229,11 +288,47 @@ class DeployDriver:
 
         #
 
-        deploy_conf_dir = self.render_deploy_path(self._deploys.CONFS_DEPLOY_DIR)
-        if app.conf is not None:
+        await self._drive_app_configure(pa)
+
+    async def _drive_app_link(
+            self,
+            app: DeployApp,
+            current_link: str,
+    ) -> None:
+        app_link = os.path.join(abs_real_path(current_link), 'apps', app.s)
+        check.state(os.path.islink(app_link))
+
+        app_dir = abs_real_path(app_link)
+        check.state(os.path.isdir(app_dir))
+
+        #
+
+        pa = await self._apps.prepare_app_link(
+            self.tags,
+            app_dir,
+        )
+
+        #
+
+        relative_symlink(
+            app_dir,
+            os.path.join(self.dir, 'apps', app.s),
+            target_is_directory=True,
+        )
+
+        #
+
+        await self._drive_app_configure(pa)
+
+    async def _drive_app_configure(
+            self,
+            pa: DeployAppManager.PreparedApp,
+    ) -> None:
+        deploy_conf_dir = self.render_path(self._deploys.CONFS_DEPLOY_DIR)
+        if pa.spec.conf is not None:
             await self._conf.link_app_conf(
-                app.conf,
-                app_tags,
-                check.non_empty_str(da.conf_dir),
+                pa.spec.conf,
+                pa.tags,
+                check.non_empty_str(pa.conf_dir),
                 deploy_conf_dir,
             )

{ominfra-0.0.0.dev190 → ominfra-0.0.0.dev192}/ominfra/manage/deploy/git.py

@@ -91,7 +91,9 @@ class DeployGitManager(SingleDirDeployPathOwner):
 
         async def fetch(self, rev: DeployRev) -> None:
             await self.init()
-            await self._call('git', 'fetch', '--depth=1', 'origin', rev)
+
+            # This fetch shouldn't be depth=1 - git doesn't reuse local data with shallow fetches.
+            await self._call('git', 'fetch', 'origin', rev)
 
         #
 

{ominfra-0.0.0.dev190 → ominfra-0.0.0.dev192}/ominfra/manage/deploy/specs.py

@@ -91,6 +91,15 @@ class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
         return DeployAppKey(self._key_str())
 
 
+@dc.dataclass(frozen=True)
+class DeployAppLinksSpec:
+    apps: ta.Sequence[DeployApp] = ()
+
+    removed_apps: ta.Sequence[DeployApp] = ()
+
+    exclude_unspecified: bool = False
+
+
 ##
 
 
@@ -109,6 +118,8 @@ class DeploySpec(DeploySpecKeyed[DeployKey]):
 
     apps: ta.Sequence[DeployAppSpec] = ()
 
+    app_links: DeployAppLinksSpec = DeployAppLinksSpec()
+
     systemd: ta.Optional[DeploySystemdSpec] = None
 
     def __post_init__(self) -> None: