ominfra 0.0.0.dev158__py3-none-any.whl → 0.0.0.dev160__py3-none-any.whl

ominfra/manage/deploy/apps.py

@@ -12,6 +12,7 @@ from .paths import DeployPathOwner
 from .specs import DeploySpec
 from .types import DeployAppTag
 from .types import DeployHome
+from .types import DeployKey
 from .types import DeployRev
 from .types import DeployTag
 from .venvs import DeployVenvManager
@@ -19,13 +20,15 @@ from .venvs import DeployVenvManager
 
 def make_deploy_tag(
         rev: DeployRev,
-        now: ta.Optional[datetime.datetime] = None,
+        key: DeployKey,
+        *,
+        utcnow: ta.Optional[datetime.datetime] = None,
 ) -> DeployTag:
-    if now is None:
-        now = datetime.datetime.utcnow()  # noqa
-    now_fmt = '%Y%m%dT%H%M%S'
-    now_str = now.strftime(now_fmt)
-    return DeployTag('-'.join([now_str, rev]))
+    if utcnow is None:
+        utcnow = datetime.datetime.now(tz=datetime.timezone.utc)  # noqa
+    now_fmt = '%Y%m%dT%H%M%SZ'
+    now_str = utcnow.strftime(now_fmt)
+    return DeployTag('-'.join([now_str, rev, key]))
 
 
 class DeployAppManager(DeployPathOwner):
@@ -46,7 +49,7 @@ class DeployAppManager(DeployPathOwner):
     def _dir(self) -> str:
         return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
 
-    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
         return {
             DeployPath.parse('apps/@app/@tag'),
         }
@@ -54,15 +57,14 @@ class DeployAppManager(DeployPathOwner):
     async def prepare_app(
             self,
             spec: DeploySpec,
-    ):
-        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev))
+    ) -> None:
+        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.checkout.rev, spec.key()))
         app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
 
         #
 
         await self._git.checkout(
-            spec.repo,
-            spec.rev,
+            spec.checkout,
             app_dir,
         )
 

ominfra/manage/deploy/atomics.py

@@ -0,0 +1,207 @@
+# ruff: noqa: UP006 UP007
+import abc
+import os
+import shutil
+import tempfile
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.strings import attr_repr
+
+
+DeployAtomicPathSwapKind = ta.Literal['dir', 'file']
+DeployAtomicPathSwapState = ta.Literal['open', 'committed', 'aborted']  # ta.TypeAlias
+
+
+##
+
+
+class DeployAtomicPathSwap(abc.ABC):
+    def __init__(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            *,
+            auto_commit: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self._kind = kind
+        self._dst_path = dst_path
+        self._auto_commit = auto_commit
+
+        self._state: DeployAtomicPathSwapState = 'open'
+
+    def __repr__(self) -> str:
+        return attr_repr(self, 'kind', 'dst_path', 'tmp_path')
+
+    @property
+    def kind(self) -> DeployAtomicPathSwapKind:
+        return self._kind
+
+    @property
+    def dst_path(self) -> str:
+        return self._dst_path
+
+    @property
+    @abc.abstractmethod
+    def tmp_path(self) -> str:
+        raise NotImplementedError
+
+    #
+
+    @property
+    def state(self) -> DeployAtomicPathSwapState:
+        return self._state
+
+    def _check_state(self, *states: DeployAtomicPathSwapState) -> None:
+        if self._state not in states:
+            raise RuntimeError(f'Atomic path swap not in correct state: {self._state}, {states}')
+
+    #
+
+    @abc.abstractmethod
+    def _commit(self) -> None:
+        raise NotImplementedError
+
+    def commit(self) -> None:
+        if self._state == 'committed':
+            return
+        self._check_state('open')
+        try:
+            self._commit()
+        except Exception:  # noqa
+            self._abort()
+            raise
+        else:
+            self._state = 'committed'
+
+    #
+
+    @abc.abstractmethod
+    def _abort(self) -> None:
+        raise NotImplementedError
+
+    def abort(self) -> None:
+        if self._state == 'aborted':
+            return
+        self._abort()
+        self._state = 'aborted'
+
+    #
+
+    def __enter__(self) -> 'DeployAtomicPathSwap':
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if (
+                exc_type is None and
+                self._auto_commit and
+                self._state == 'open'
+        ):
+            self.commit()
+        else:
+            self.abort()
+
+
+#
+
+
+class DeployAtomicPathSwapping(abc.ABC):
+    @abc.abstractmethod
+    def begin_atomic_path_swap(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            *,
+            name_hint: ta.Optional[str] = None,
+            make_dirs: bool = False,
+            **kwargs: ta.Any,
+    ) -> DeployAtomicPathSwap:
+        raise NotImplementedError
+
+
+##
+
+
+class OsRenameDeployAtomicPathSwap(DeployAtomicPathSwap):
+    def __init__(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            tmp_path: str,
+            **kwargs: ta.Any,
+    ) -> None:
+        if kind == 'dir':
+            check.state(os.path.isdir(tmp_path))
+        elif kind == 'file':
+            check.state(os.path.isfile(tmp_path))
+        else:
+            raise TypeError(kind)
+
+        super().__init__(
+            kind,
+            dst_path,
+            **kwargs,
+        )
+
+        self._tmp_path = tmp_path
+
+    @property
+    def tmp_path(self) -> str:
+        return self._tmp_path
+
+    def _commit(self) -> None:
+        os.rename(self._tmp_path, self._dst_path)
+
+    def _abort(self) -> None:
+        shutil.rmtree(self._tmp_path, ignore_errors=True)
+
+
+class TempDirDeployAtomicPathSwapping(DeployAtomicPathSwapping):
+    def __init__(
+            self,
+            *,
+            temp_dir: ta.Optional[str] = None,
+            root_dir: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+
+        if root_dir is not None:
+            root_dir = os.path.abspath(root_dir)
+        self._root_dir = root_dir
+        self._temp_dir = temp_dir
+
+    def begin_atomic_path_swap(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            *,
+            name_hint: ta.Optional[str] = None,
+            make_dirs: bool = False,
+            **kwargs: ta.Any,
+    ) -> DeployAtomicPathSwap:
+        dst_path = os.path.abspath(dst_path)
+        if self._root_dir is not None and not dst_path.startswith(check.non_empty_str(self._root_dir)):
+            raise RuntimeError(f'Atomic path swap dst must be in root dir: {dst_path}, {self._root_dir}')
+
+        dst_dir = os.path.dirname(dst_path)
+        if make_dirs:
+            os.makedirs(dst_dir, exist_ok=True)
+        if not os.path.isdir(dst_dir):
+            raise RuntimeError(f'Atomic path swap dst dir does not exist: {dst_dir}')
+
+        if kind == 'dir':
+            tmp_path = tempfile.mkdtemp(prefix=name_hint, dir=self._temp_dir)
+        elif kind == 'file':
+            fd, tmp_path = tempfile.mkstemp(prefix=name_hint, dir=self._temp_dir)
+            os.close(fd)
+        else:
+            raise TypeError(kind)
+
+        return OsRenameDeployAtomicPathSwap(
+            kind,
+            dst_path,
+            tmp_path,
+            **kwargs,
+        )

ominfra/manage/deploy/commands.py

@@ -5,6 +5,8 @@ from omlish.lite.logs import log
 
 from ..commands.base import Command
 from ..commands.base import CommandExecutor
+from .apps import DeployAppManager
+from .specs import DeploySpec
 
 
 ##
@@ -12,13 +14,20 @@ from ..commands.base import CommandExecutor
 
 @dc.dataclass(frozen=True)
 class DeployCommand(Command['DeployCommand.Output']):
+    spec: DeploySpec
+
     @dc.dataclass(frozen=True)
     class Output(Command.Output):
         pass
 
 
+@dc.dataclass(frozen=True)
 class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
+    _apps: DeployAppManager
+
     async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
-        log.info('Deploying!')
+        log.info('Deploying! %r', cmd.spec)
+
+        await self._apps.prepare_app(cmd.spec)
 
         return DeployCommand.Output()

ominfra/manage/deploy/git.py

@@ -14,11 +14,11 @@ import typing as ta
 
 from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.cached import async_cached_nullary
-from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 
-from .paths import DeployPath
-from .paths import DeployPathOwner
+from .atomics import DeployAtomicPathSwapping
+from .paths import SingleDirDeployPathOwner
+from .specs import DeployGitCheckout
 from .specs import DeployGitRepo
 from .types import DeployHome
 from .types import DeployRev
@@ -27,27 +27,22 @@ from .types import DeployRev
 ##
 
 
-class DeployGitManager(DeployPathOwner):
+class DeployGitManager(SingleDirDeployPathOwner):
     def __init__(
             self,
             *,
             deploy_home: ta.Optional[DeployHome] = None,
+            atomics: DeployAtomicPathSwapping,
     ) -> None:
-        super().__init__()
+        super().__init__(
+            owned_dir='git',
+            deploy_home=deploy_home,
+        )
 
-        self._deploy_home = deploy_home
+        self._atomics = atomics
 
         self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
 
-    @cached_nullary
-    def _dir(self) -> str:
-        return os.path.join(check.non_empty_str(self._deploy_home), 'git')
-
-    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
-        return {
-            DeployPath.parse('git'),
-        }
-
     class RepoDir:
         def __init__(
                 self,
@@ -59,7 +54,7 @@ class DeployGitManager(DeployPathOwner):
             self._git = git
             self._repo = repo
             self._dir = os.path.join(
-                self._git._dir(),  # noqa
+                self._git._make_dir(),  # noqa
                 check.non_empty_str(repo.host),
                 check.non_empty_str(repo.path),
             )
@@ -75,12 +70,16 @@ class DeployGitManager(DeployPathOwner):
             else:
                 return f'https://{self._repo.host}/{self._repo.path}'
 
+        #
+
         async def _call(self, *cmd: str) -> None:
             await asyncio_subprocesses.check_call(
                 *cmd,
                 cwd=self._dir,
             )
 
+        #
+
         @async_cached_nullary
         async def init(self) -> None:
             os.makedirs(self._dir, exist_ok=True)
@@ -94,20 +93,24 @@ class DeployGitManager(DeployPathOwner):
             await self.init()
             await self._call('git', 'fetch', '--depth=1', 'origin', rev)
 
-        async def checkout(self, rev: DeployRev, dst_dir: str) -> None:
-            check.state(not os.path.exists(dst_dir))
-
-            await self.fetch(rev)
+        #
 
-            # FIXME: temp dir swap
-            os.makedirs(dst_dir)
+        async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
+            check.state(not os.path.exists(dst_dir))
+            with self._git._atomics.begin_atomic_path_swap(  # noqa
+                    'dir',
+                    dst_dir,
+                    auto_commit=True,
+                    make_dirs=True,
+            ) as dst_swap:
+                await self.fetch(checkout.rev)
 
-            dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_dir)
-            await dst_call('git', 'init')
+                dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
+                await dst_call('git', 'init')
 
-            await dst_call('git', 'remote', 'add', 'local', self._dir)
-            await dst_call('git', 'fetch', '--depth=1', 'local', rev)
-            await dst_call('git', 'checkout', rev)
+                await dst_call('git', 'remote', 'add', 'local', self._dir)
+                await dst_call('git', 'fetch', '--depth=1', 'local', checkout.rev)
+                await dst_call('git', 'checkout', checkout.rev, *(checkout.subtrees or []))
 
     def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
         try:
@@ -116,5 +119,5 @@ class DeployGitManager(DeployPathOwner):
             repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
             return repo_dir
 
-    async def checkout(self, repo: DeployGitRepo, rev: DeployRev, dst_dir: str) -> None:
-        await self.get_repo_dir(repo).checkout(rev, dst_dir)
+    async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
+        await self.get_repo_dir(checkout.repo).checkout(checkout, dst_dir)

ominfra/manage/deploy/inject.py

@@ -8,12 +8,14 @@ from omlish.lite.inject import inj
 
 from ..commands.inject import bind_command
 from .apps import DeployAppManager
+from .atomics import DeployAtomicPathSwapping
 from .commands import DeployCommand
 from .commands import DeployCommandExecutor
 from .config import DeployConfig
 from .git import DeployGitManager
 from .interp import InterpCommand
 from .interp import InterpCommandExecutor
+from .tmp import DeployTmpManager
 from .types import DeployHome
 from .venvs import DeployVenvManager
 
@@ -25,10 +27,19 @@ def bind_deploy(
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(deploy_config),
 
+        #
+
         inj.bind(DeployAppManager, singleton=True),
+
         inj.bind(DeployGitManager, singleton=True),
+
+        inj.bind(DeployTmpManager, singleton=True),
+        inj.bind(DeployAtomicPathSwapping, to_key=DeployTmpManager),
+
         inj.bind(DeployVenvManager, singleton=True),
 
+        #
+
         bind_command(DeployCommand, DeployCommandExecutor),
         bind_command(InterpCommand, InterpCommandExecutor),
     ]

ominfra/manage/deploy/paths.py

@@ -1,46 +1,22 @@
 # ruff: noqa: UP006 UP007
 """
-~deploy
-  deploy.pid (flock)
-  /app
-    /<appplaceholder> - shallow clone
-  /conf
-    /env
-      <appplaceholder>.env
-    /nginx
-      <appplaceholder>.conf
-    /supervisor
-      <appplaceholder>.conf
-  /venv
-    /<appplaceholder>
-
-?
-  /logs
-    /wrmsr--omlish--<placeholder>
-
-placeholder = <name>--<rev>--<when>
-
-==
-
-for dn in [
-    'app',
-    'conf',
-    'conf/env',
-    'conf/nginx',
-    'conf/supervisor',
-    'venv',
-]:
-
-==
-
+TODO:
+ - run/pidfile
+ - logs/...
+ - current symlink
+ - conf/{nginx,supervisor}
+ - env/?
 """
 import abc
 import dataclasses as dc
 import os.path
 import typing as ta
 
+from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 
+from .types import DeployHome
+
 
 DeployPathKind = ta.Literal['dir', 'file']  # ta.TypeAlias
 DeployPathPlaceholder = ta.Literal['app', 'tag']  # ta.TypeAlias
@@ -54,7 +30,7 @@ DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
 
 DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
     'app',
-    'tag',  # <rev>-<dt>
+    'tag',
 ])
 
 
@@ -181,6 +157,8 @@ class DeployPath:
     parts: ta.Sequence[DeployPathPart]
 
     def __post_init__(self) -> None:
+        hash(self)
+
         check.not_empty(self.parts)
         for p in self.parts[:-1]:
             check.equal(p.kind, 'dir')
@@ -215,10 +193,10 @@ class DeployPath:
         else:
             tail_parse = FileDeployPathPart.parse
         ps = check.non_empty_str(s).split('/')
-        return cls([
+        return cls((
             *([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
             tail_parse(ps[-1]),
-        ])
+        ))
 
 
 ##
@@ -226,5 +204,36 @@ class DeployPath:
 
 class DeployPathOwner(abc.ABC):
     @abc.abstractmethod
-    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
         raise NotImplementedError
+
+
+class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
+    def __init__(
+            self,
+            *args: ta.Any,
+            owned_dir: str,
+            deploy_home: ta.Optional[DeployHome],
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+
+        check.not_in('/', owned_dir)
+        self._owned_dir: str = check.non_empty_str(owned_dir)
+
+        self._deploy_home = deploy_home
+
+        self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
+
+    @cached_nullary
+    def _dir(self) -> str:
+        return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
+
+    @cached_nullary
+    def _make_dir(self) -> str:
+        if not os.path.isdir(d := self._dir()):
+            os.makedirs(d, exist_ok=True)
+        return d
+
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return self._owned_deploy_paths

ominfra/manage/deploy/specs.py

@@ -1,10 +1,13 @@
 # ruff: noqa: UP006 UP007
 import dataclasses as dc
+import hashlib
 import typing as ta
 
+from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 
 from .types import DeployApp
+from .types import DeployKey
 from .types import DeployRev
 
 
@@ -22,11 +25,32 @@ class DeployGitRepo:
         check.not_in('.', check.non_empty_str(self.path))
 
 
+@dc.dataclass(frozen=True)
+class DeployGitCheckout:
+    repo: DeployGitRepo
+    rev: DeployRev
+
+    subtrees: ta.Optional[ta.Sequence[str]] = None
+
+    def __post_init__(self) -> None:
+        hash(self)
+        check.non_empty_str(self.rev)
+        if self.subtrees is not None:
+            for st in self.subtrees:
+                check.non_empty_str(st)
+
+
 ##
 
 
 @dc.dataclass(frozen=True)
 class DeploySpec:
     app: DeployApp
-    repo: DeployGitRepo
-    rev: DeployRev
+    checkout: DeployGitCheckout
+
+    def __post_init__(self) -> None:
+        hash(self)
+
+    @cached_nullary
+    def key(self) -> DeployKey:
+        return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])

ominfra/manage/deploy/tmp.py

@@ -0,0 +1,46 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+
+from .atomics import DeployAtomicPathSwap
+from .atomics import DeployAtomicPathSwapKind
+from .atomics import DeployAtomicPathSwapping
+from .atomics import TempDirDeployAtomicPathSwapping
+from .paths import SingleDirDeployPathOwner
+from .types import DeployHome
+
+
+class DeployTmpManager(
+    SingleDirDeployPathOwner,
+    DeployAtomicPathSwapping,
+):
+    def __init__(
+            self,
+            *,
+            deploy_home: ta.Optional[DeployHome] = None,
+    ) -> None:
+        super().__init__(
+            owned_dir='tmp',
+            deploy_home=deploy_home,
+        )
+
+    @cached_nullary
+    def _swapping(self) -> DeployAtomicPathSwapping:
+        return TempDirDeployAtomicPathSwapping(
+            temp_dir=self._make_dir(),
+            root_dir=check.non_empty_str(self._deploy_home),
+        )
+
+    def begin_atomic_path_swap(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            **kwargs: ta.Any,
+    ) -> DeployAtomicPathSwap:
+        return self._swapping().begin_atomic_path_swap(
+            kind,
+            dst_path,
+            **kwargs,
+        )

ominfra/manage/deploy/types.py

@@ -6,6 +6,7 @@ DeployHome = ta.NewType('DeployHome', str)
 DeployApp = ta.NewType('DeployApp', str)
 DeployTag = ta.NewType('DeployTag', str)
 DeployRev = ta.NewType('DeployRev', str)
+DeployKey = ta.NewType('DeployKey', str)
 
 
 class DeployAppTag(ta.NamedTuple):