ominfra 0.0.0.dev171__tar.gz → 0.0.0.dev173__tar.gz

{ominfra-0.0.0.dev171/ominfra.egg-info → ominfra-0.0.0.dev173}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev171
+Version: 0.0.0.dev173
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev171
-Requires-Dist: omlish==0.0.0.dev171
+Requires-Dist: omdev==0.0.0.dev173
+Requires-Dist: omlish==0.0.0.dev173
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.18; extra == "all"

ominfra-0.0.0.dev173/ominfra/manage/deploy/apps.py

@@ -0,0 +1,176 @@
+# ruff: noqa: UP006 UP007
+import os.path
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.os.paths import relative_symlink
+
+from .conf import DeployConfManager
+from .git import DeployGitManager
+from .paths.owners import DeployPathOwner
+from .paths.paths import DeployPath
+from .specs import DeployAppSpec
+from .tags import DeployTagMap
+from .types import DeployHome
+from .venvs import DeployVenvManager
+
+
+class DeployAppManager(DeployPathOwner):
+    def __init__(
+            self,
+            *,
+            deploy_home: ta.Optional[DeployHome] = None,
+
+            conf: DeployConfManager,
+            git: DeployGitManager,
+            venvs: DeployVenvManager,
+    ) -> None:
+        super().__init__()
+
+        self._deploy_home = deploy_home
+
+        self._conf = conf
+        self._git = git
+        self._venvs = venvs
+
+    #
+
+    _APP_DIR_STR = 'apps/@app/@time--@app-rev--@app-key/'
+    _APP_DIR = DeployPath.parse(_APP_DIR_STR)
+
+    _DEPLOY_DIR_STR = 'deploys/@time--@deploy-key/'
+    _DEPLOY_DIR = DeployPath.parse(_DEPLOY_DIR_STR)
+
+    _APP_DEPLOY_LINK = DeployPath.parse(f'{_DEPLOY_DIR_STR}apps/@app')
+    _CONF_DEPLOY_DIR = DeployPath.parse(f'{_DEPLOY_DIR_STR}conf/@conf/')
+
+    @cached_nullary
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return {
+            self._APP_DIR,
+
+            self._DEPLOY_DIR,
+
+            self._APP_DEPLOY_LINK,
+            self._CONF_DEPLOY_DIR,
+
+            *[
+                DeployPath.parse(f'{self._APP_DIR_STR}{sfx}/')
+                for sfx in [
+                    'conf',
+                    'git',
+                    'venv',
+                ]
+            ],
+        }
+
+    #
+
+    async def prepare_app(
+            self,
+            spec: DeployAppSpec,
+            tags: DeployTagMap,
+    ) -> None:
+        deploy_home = check.non_empty_str(self._deploy_home)
+
+        def build_path(pth: DeployPath) -> str:
+            return os.path.join(deploy_home, pth.render(tags))
+
+        app_dir = build_path(self._APP_DIR)
+        deploy_dir = build_path(self._DEPLOY_DIR)
+        app_deploy_link = build_path(self._APP_DEPLOY_LINK)
+
+        #
+
+        os.makedirs(deploy_dir, exist_ok=True)
+
+        deploying_link = os.path.join(deploy_home, 'deploys/deploying')
+        relative_symlink(
+            deploy_dir,
+            deploying_link,
+            target_is_directory=True,
+            make_dirs=True,
+        )
+
+        #
+
+        os.makedirs(app_dir)
+        relative_symlink(
+            app_dir,
+            app_deploy_link,
+            target_is_directory=True,
+            make_dirs=True,
+        )
+
+        #
+
+        deploy_conf_dir = os.path.join(deploy_dir, 'conf')
+        os.makedirs(deploy_conf_dir, exist_ok=True)
+
+        #
+
+        # def mirror_symlinks(src: str, dst: str) -> None:
+        #     def mirror_link(lp: str) -> None:
+        #         check.state(os.path.islink(lp))
+        #         shutil.copy2(
+        #             lp,
+        #             os.path.join(dst, os.path.relpath(lp, src)),
+        #             follow_symlinks=False,
+        #         )
+        #
+        #     for dp, dns, fns in os.walk(src, followlinks=False):
+        #         for fn in fns:
+        #             mirror_link(os.path.join(dp, fn))
+        #
+        #         for dn in dns:
+        #             dp2 = os.path.join(dp, dn)
+        #             if os.path.islink(dp2):
+        #                 mirror_link(dp2)
+        #             else:
+        #                 os.makedirs(os.path.join(dst, os.path.relpath(dp2, src)))
+
+        current_link = os.path.join(deploy_home, 'deploys/current')
+
+        # if os.path.exists(current_link):
+        #     mirror_symlinks(
+        #         os.path.join(current_link, 'conf'),
+        #         conf_tag_dir,
+        #     )
+        #     mirror_symlinks(
+        #         os.path.join(current_link, 'apps'),
+        #         os.path.join(deploy_dir, 'apps'),
+        #     )
+
+        #
+
+        app_git_dir = os.path.join(app_dir, 'git')
+        await self._git.checkout(
+            spec.git,
+            app_git_dir,
+        )
+
+        #
+
+        if spec.venv is not None:
+            app_venv_dir = os.path.join(app_dir, 'venv')
+            await self._venvs.setup_venv(
+                spec.venv,
+                app_git_dir,
+                app_venv_dir,
+            )
+
+        #
+
+        if spec.conf is not None:
+            app_conf_dir = os.path.join(app_dir, 'conf')
+            await self._conf.write_app_conf(
+                spec.conf,
+                tags,
+                app_conf_dir,
+                deploy_conf_dir,
+            )
+
+        #
+
+        os.replace(deploying_link, current_link)

{ominfra-0.0.0.dev171 → ominfra-0.0.0.dev173}/ominfra/manage/deploy/conf.py

@@ -23,13 +23,15 @@ from omlish.lite.check import check
 from omlish.os.paths import is_path_in_dir
 from omlish.os.paths import relative_symlink
 
-from .paths.paths import DEPLOY_PATH_PLACEHOLDER_SEPARATOR
-from .specs import AppDeployConfLink
-from .specs import DeployConfFile
-from .specs import DeployConfLink
-from .specs import DeployConfSpec
-from .specs import TagDeployConfLink
-from .types import DeployAppTag
+from .paths.paths import DeployPath
+from .specs import AllActiveDeployAppConfLink
+from .specs import CurrentOnlyDeployAppConfLink
+from .specs import DeployAppConfFile
+from .specs import DeployAppConfLink
+from .specs import DeployAppConfSpec
+from .tags import DEPLOY_TAG_SEPARATOR
+from .tags import DeployApp
+from .tags import DeployTagMap
 from .types import DeployHome
 
 
@@ -45,18 +47,18 @@ class DeployConfManager:
 
     #
 
-    async def _write_conf_file(
+    async def _write_app_conf_file(
             self,
-            cf: DeployConfFile,
-            conf_dir: str,
+            acf: DeployAppConfFile,
+            app_conf_dir: str,
     ) -> None:
-        conf_file = os.path.join(conf_dir, cf.path)
-        check.arg(is_path_in_dir(conf_dir, conf_file))
+        conf_file = os.path.join(app_conf_dir, acf.path)
+        check.arg(is_path_in_dir(app_conf_dir, conf_file))
 
         os.makedirs(os.path.dirname(conf_file), exist_ok=True)
 
         with open(conf_file, 'w') as f:  # noqa
-            f.write(cf.body)
+            f.write(acf.body)
 
     #
 
@@ -65,15 +67,18 @@ class DeployConfManager:
         link_src: str
         link_dst: str
 
-    def _compute_conf_link_dst(
+    _UNIQUE_LINK_NAME_STR = '@app--@time--@app-key'
+    _UNIQUE_LINK_NAME = DeployPath.parse(_UNIQUE_LINK_NAME_STR)
+
+    def _compute_app_conf_link_dst(
             self,
-            link: DeployConfLink,
-            app_tag: DeployAppTag,
-            conf_dir: str,
-            link_dir: str,
+            link: DeployAppConfLink,
+            tags: DeployTagMap,
+            app_conf_dir: str,
+            conf_link_dir: str,
     ) -> _ComputedConfLink:
-        link_src = os.path.join(conf_dir, link.src)
-        check.arg(is_path_in_dir(conf_dir, link_src))
+        link_src = os.path.join(app_conf_dir, link.src)
+        check.arg(is_path_in_dir(app_conf_dir, link_src))
 
         #
 
@@ -88,7 +93,7 @@ class DeployConfManager:
             d, f = os.path.split(link.src)
             # TODO: check filename :|
             link_dst_pfx = d + '/'
-            link_dst_sfx = DEPLOY_PATH_PLACEHOLDER_SEPARATOR + f
+            link_dst_sfx = DEPLOY_TAG_SEPARATOR + f
 
         else:  # noqa
             # @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
@@ -102,10 +107,10 @@ class DeployConfManager:
 
         #
 
-        if isinstance(link, AppDeployConfLink):
-            link_dst_mid = str(app_tag.app)
-        elif isinstance(link, TagDeployConfLink):
-            link_dst_mid = DEPLOY_PATH_PLACEHOLDER_SEPARATOR.join([app_tag.app, app_tag.tag])
+        if isinstance(link, CurrentOnlyDeployAppConfLink):
+            link_dst_mid = str(tags[DeployApp].s)
+        elif isinstance(link, AllActiveDeployAppConfLink):
+            link_dst_mid = self._UNIQUE_LINK_NAME.render(tags)
         else:
             raise TypeError(link)
 
@@ -116,7 +121,7 @@ class DeployConfManager:
             link_dst_mid,
             link_dst_sfx,
         ])
-        link_dst = os.path.join(link_dir, link_dst_name)
+        link_dst = os.path.join(conf_link_dir, link_dst_name)
 
         return DeployConfManager._ComputedConfLink(
             is_dir=is_dir,
@@ -124,24 +129,24 @@ class DeployConfManager:
             link_dst=link_dst,
         )
 
-    async def _make_conf_link(
+    async def _make_app_conf_link(
             self,
-            link: DeployConfLink,
-            app_tag: DeployAppTag,
-            conf_dir: str,
-            link_dir: str,
+            link: DeployAppConfLink,
+            tags: DeployTagMap,
+            app_conf_dir: str,
+            conf_link_dir: str,
     ) -> None:
-        comp = self._compute_conf_link_dst(
+        comp = self._compute_app_conf_link_dst(
             link,
-            app_tag,
-            conf_dir,
-            link_dir,
+            tags,
+            app_conf_dir,
+            conf_link_dir,
         )
 
         #
 
-        check.arg(is_path_in_dir(conf_dir, comp.link_src))
-        check.arg(is_path_in_dir(link_dir, comp.link_dst))
+        check.arg(is_path_in_dir(app_conf_dir, comp.link_src))
+        check.arg(is_path_in_dir(conf_link_dir, comp.link_dst))
 
         if comp.is_dir:
             check.arg(os.path.isdir(comp.link_src))
@@ -159,25 +164,25 @@ class DeployConfManager:
 
     #
 
-    async def write_conf(
+    async def write_app_conf(
             self,
-            spec: DeployConfSpec,
-            app_tag: DeployAppTag,
-            conf_dir: str,
-            link_dir: str,
+            spec: DeployAppConfSpec,
+            tags: DeployTagMap,
+            app_conf_dir: str,
+            conf_link_dir: str,
     ) -> None:
-        for cf in spec.files or []:
-            await self._write_conf_file(
-                cf,
-                conf_dir,
+        for acf in spec.files or []:
+            await self._write_app_conf_file(
+                acf,
+                app_conf_dir,
             )
 
         #
 
         for link in spec.links or []:
-            await self._make_conf_link(
+            await self._make_app_conf_link(
                 link,
-                app_tag,
-                conf_dir,
-                link_dir,
+                tags,
+                app_conf_dir,
+                conf_link_dir,
             )

ominfra-0.0.0.dev173/ominfra/manage/deploy/deploy.py

@@ -0,0 +1,71 @@
+# ruff: noqa: UP006 UP007
+import datetime
+import typing as ta
+
+from omlish.lite.typing import Func0
+
+from .apps import DeployAppManager
+from .paths.manager import DeployPathsManager
+from .specs import DeploySpec
+from .tags import DeployAppRev
+from .tags import DeployTagMap
+from .tags import DeployTime
+
+
+DEPLOY_TAG_DATETIME_FMT = '%Y%m%dT%H%M%SZ'
+
+
+DeployManagerUtcClock = ta.NewType('DeployManagerUtcClock', Func0[datetime.datetime])
+
+
+class DeployManager:
+    def __init__(
+            self,
+            *,
+            apps: DeployAppManager,
+            paths: DeployPathsManager,
+
+            utc_clock: ta.Optional[DeployManagerUtcClock] = None,
+    ):
+        super().__init__()
+
+        self._apps = apps
+        self._paths = paths
+
+        self._utc_clock = utc_clock
+
+    def _utc_now(self) -> datetime.datetime:
+        if self._utc_clock is not None:
+            return self._utc_clock()  # noqa
+        else:
+            return datetime.datetime.now(tz=datetime.timezone.utc)  # noqa
+
+    def _make_deploy_time(self) -> DeployTime:
+        return DeployTime(self._utc_now().strftime(DEPLOY_TAG_DATETIME_FMT))
+
+    async def run_deploy(
+            self,
+            spec: DeploySpec,
+    ) -> None:
+        self._paths.validate_deploy_paths()
+
+        #
+
+        deploy_tags = DeployTagMap(
+            self._make_deploy_time(),
+            spec.key(),
+        )
+
+        #
+
+        for app in spec.apps:
+            app_tags = deploy_tags.add(
+                app.app,
+                app.key(),
+                DeployAppRev(app.git.rev),
+            )
+
+            await self._apps.prepare_app(
+                app,
+                app_tags,
+            )

{ominfra-0.0.0.dev171 → ominfra-0.0.0.dev173}/ominfra/manage/deploy/paths/paths.py

@@ -13,38 +13,28 @@ import dataclasses as dc
 import itertools
 import typing as ta
 
+from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 from omlish.lite.strings import split_keep_delimiter
 
-from ..types import DeployPathKind
-from ..types import DeployPathPlaceholder
+from ..tags import DEPLOY_TAG_DELIMITERS
+from ..tags import DEPLOY_TAG_SIGIL
+from ..tags import DEPLOY_TAGS_BY_NAME
+from ..tags import DeployTag
+from ..tags import DeployTagMap
+from .types import DeployPathKind
 
 
 ##
 
 
-DEPLOY_PATH_PLACEHOLDER_SIGIL = '@'
-DEPLOY_PATH_PLACEHOLDER_SEPARATOR = '--'
-
-DEPLOY_PATH_PLACEHOLDER_DELIMITERS: ta.AbstractSet[str] = frozenset([
-    DEPLOY_PATH_PLACEHOLDER_SEPARATOR,
-    '.',
-])
-
-DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
-    'app',
-    'tag',
-    'conf',
-])
-
-
 class DeployPathError(Exception):
     pass
 
 
 class DeployPathRenderable(abc.ABC):
     @abc.abstractmethod
-    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+    def render(self, tags: ta.Optional[DeployTagMap] = None) -> str:
         raise NotImplementedError
 
 
@@ -55,26 +45,30 @@ class DeployPathNamePart(DeployPathRenderable, abc.ABC):
     @classmethod
     def parse(cls, s: str) -> 'DeployPathNamePart':
         check.non_empty_str(s)
-        if s.startswith(DEPLOY_PATH_PLACEHOLDER_SIGIL):
-            return PlaceholderDeployPathNamePart(s[1:])
-        elif s in DEPLOY_PATH_PLACEHOLDER_DELIMITERS:
+        if s.startswith(DEPLOY_TAG_SIGIL):
+            return TagDeployPathNamePart(s[1:])
+        elif s in DEPLOY_TAG_DELIMITERS:
             return DelimiterDeployPathNamePart(s)
         else:
             return ConstDeployPathNamePart(s)
 
 
 @dc.dataclass(frozen=True)
-class PlaceholderDeployPathNamePart(DeployPathNamePart):
-    placeholder: str  # DeployPathPlaceholder
+class TagDeployPathNamePart(DeployPathNamePart):
+    name: str
 
     def __post_init__(self) -> None:
-        check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
+        check.in_(self.name, DEPLOY_TAGS_BY_NAME)
 
-    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
-        if placeholders is not None:
-            return placeholders[self.placeholder]  # type: ignore
+    @property
+    def tag(self) -> ta.Type[DeployTag]:
+        return DEPLOY_TAGS_BY_NAME[self.name]
+
+    def render(self, tags: ta.Optional[DeployTagMap] = None) -> str:
+        if tags is not None:
+            return tags[self.tag].s
         else:
-            return DEPLOY_PATH_PLACEHOLDER_SIGIL + self.placeholder
+            return DEPLOY_TAG_SIGIL + self.name
 
 
 @dc.dataclass(frozen=True)
@@ -82,9 +76,9 @@ class DelimiterDeployPathNamePart(DeployPathNamePart):
     delimiter: str
 
     def __post_init__(self) -> None:
-        check.in_(self.delimiter, DEPLOY_PATH_PLACEHOLDER_DELIMITERS)
+        check.in_(self.delimiter, DEPLOY_TAG_DELIMITERS)
 
-    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+    def render(self, tags: ta.Optional[DeployTagMap] = None) -> str:
         return self.delimiter
 
 
@@ -94,10 +88,10 @@ class ConstDeployPathNamePart(DeployPathNamePart):
 
     def __post_init__(self) -> None:
         check.non_empty_str(self.const)
-        for c in [*DEPLOY_PATH_PLACEHOLDER_DELIMITERS, DEPLOY_PATH_PLACEHOLDER_SIGIL, '/']:
+        for c in [*DEPLOY_TAG_DELIMITERS, DEPLOY_TAG_SIGIL, '/']:
             check.not_in(c, self.const)
 
-    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+    def render(self, tags: ta.Optional[DeployTagMap] = None) -> str:
         return self.const
 
 
@@ -112,8 +106,8 @@ class DeployPathName(DeployPathRenderable):
             if len(gl := list(g)) > 1:
                 raise DeployPathError(f'May not have consecutive path name part types: {k} {gl}')
 
-    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
-        return ''.join(p.render(placeholders) for p in self.parts)
+    def render(self, tags: ta.Optional[DeployTagMap] = None) -> str:
+        return ''.join(p.render(tags) for p in self.parts)
 
     @classmethod
     def parse(cls, s: str) -> 'DeployPathName':
@@ -123,7 +117,7 @@ class DeployPathName(DeployPathRenderable):
         i = 0
         ps = []
         while i < len(s):
-            ns = [(n, d) for d in DEPLOY_PATH_PLACEHOLDER_DELIMITERS if (n := s.find(d, i)) >= 0]
+            ns = [(n, d) for d in DEPLOY_TAG_DELIMITERS if (n := s.find(d, i)) >= 0]
             if not ns:
                 ps.append(s[i:])
                 break
@@ -147,8 +141,8 @@ class DeployPathPart(DeployPathRenderable, abc.ABC):  # noqa
     def kind(self) -> DeployPathKind:
         raise NotImplementedError
 
-    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
-        return self.name.render(placeholders) + ('/' if self.kind == 'dir' else '')
+    def render(self, tags: ta.Optional[DeployTagMap] = None) -> str:
+        return self.name.render(tags) + ('/' if self.kind == 'dir' else '')
 
     @classmethod
     def parse(cls, s: str) -> 'DeployPathPart':
@@ -194,20 +188,20 @@ class DeployPath:
         for p in self.parts[:-1]:
             check.equal(p.kind, 'dir')
 
-        pd: ta.Dict[DeployPathPlaceholder, ta.List[int]] = {}
+    @cached_nullary
+    def tag_indices(self) -> ta.Mapping[ta.Type[DeployTag], ta.Sequence[int]]:
+        pd: ta.Dict[ta.Type[DeployTag], ta.List[int]] = {}
         for i, np in enumerate(self.name_parts):
-            if isinstance(np, PlaceholderDeployPathNamePart):
-                pd.setdefault(ta.cast(DeployPathPlaceholder, np.placeholder), []).append(i)
-
-        # if 'tag' in pd and 'app' not in pd:
-        #     raise DeployPathError('Tag placeholder in path without app', self)
+            if isinstance(np, TagDeployPathNamePart):
+                pd.setdefault(np.tag, []).append(i)
+        return pd
 
     @property
     def kind(self) -> ta.Literal['file', 'dir']:
         return self.parts[-1].kind
 
-    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
-        return ''.join([p.render(placeholders) for p in self.parts])
+    def render(self, tags: ta.Optional[DeployTagMap] = None) -> str:
+        return ''.join([p.render(tags) for p in self.parts])
 
     @classmethod
     def parse(cls, s: str) -> 'DeployPath':

ominfra-0.0.0.dev173/ominfra/manage/deploy/paths/types.py

@@ -0,0 +1,8 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+
+DeployPathKind = ta.Literal['dir', 'file']  # ta.TypeAlias
+
+
+##