ominfra 0.0.0.dev157__py3-none-any.whl → 0.0.0.dev159__py3-none-any.whl

ominfra/clouds/aws/journald2aws/main.py

@@ -5,7 +5,7 @@ import dataclasses as dc
 import os.path
 import sys
 
-from omlish.lite.logs import configure_standard_logging
+from omlish.logs.standard import configure_standard_logging
 
 from ....configs import read_config_file
 from .driver import JournalctlToAwsDriver

ominfra/journald/tailer.py

@@ -410,8 +410,8 @@ import typing as ta
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 from omlish.lite.logs import log
-from omlish.lite.subprocesses import subprocess_close
-from omlish.lite.subprocesses import subprocess_shell_wrap_exec
+from omlish.subprocesses import subprocess_close
+from omlish.subprocesses import subprocess_shell_wrap_exec
 
 from ..threadworkers import ThreadWorker
 from .messages import JournalctlMessage  # noqa

ominfra/manage/bootstrap_.py

@@ -1,7 +1,7 @@
 # ruff: noqa: UP006 UP007
 from omlish.lite.inject import Injector
 from omlish.lite.inject import inj
-from omlish.lite.logs import configure_standard_logging
+from omlish.logs.standard import configure_standard_logging
 
 from .bootstrap import MainBootstrap
 from .inject import bind_main

ominfra/manage/commands/subprocess.py

@@ -6,11 +6,11 @@ import subprocess
 import time
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
-from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
-from omlish.lite.subprocesses import SubprocessChannelOption
-from omlish.lite.subprocesses import subprocess_maybe_shell_wrap_exec
+from omlish.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
+from omlish.subprocesses import SubprocessChannelOption
+from omlish.subprocesses import subprocess_maybe_shell_wrap_exec
 
 from .base import Command
 from .base import CommandExecutor

ominfra/manage/deploy/apps.py

@@ -3,14 +3,16 @@ import datetime
 import os.path
 import typing as ta
 
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+
 from .git import DeployGitManager
-from .git import DeployGitRepo
-from .git import DeployGitSpec
 from .paths import DeployPath
 from .paths import DeployPathOwner
-from .types import DeployApp
+from .specs import DeploySpec
 from .types import DeployAppTag
 from .types import DeployHome
+from .types import DeployKey
 from .types import DeployRev
 from .types import DeployTag
 from .venvs import DeployVenvManager
@@ -18,20 +20,22 @@ from .venvs import DeployVenvManager
 
 def make_deploy_tag(
         rev: DeployRev,
-        now: ta.Optional[datetime.datetime] = None,
+        key: DeployKey,
+        *,
+        utcnow: ta.Optional[datetime.datetime] = None,
 ) -> DeployTag:
-    if now is None:
-        now = datetime.datetime.utcnow()  # noqa
-    now_fmt = '%Y%m%dT%H%M%S'
-    now_str = now.strftime(now_fmt)
-    return DeployTag('-'.join([rev, now_str]))
+    if utcnow is None:
+        utcnow = datetime.datetime.now(tz=datetime.timezone.utc)  # noqa
+    now_fmt = '%Y%m%dT%H%M%SZ'
+    now_str = utcnow.strftime(now_fmt)
+    return DeployTag('-'.join([now_str, rev, key]))
 
 
 class DeployAppManager(DeployPathOwner):
     def __init__(
             self,
             *,
-            deploy_home: DeployHome,
+            deploy_home: ta.Optional[DeployHome] = None,
             git: DeployGitManager,
             venvs: DeployVenvManager,
     ) -> None:
@@ -41,29 +45,27 @@ class DeployAppManager(DeployPathOwner):
         self._git = git
         self._venvs = venvs
 
-        self._dir = os.path.join(deploy_home, 'apps')
+    @cached_nullary
+    def _dir(self) -> str:
+        return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
 
-    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
         return {
             DeployPath.parse('apps/@app/@tag'),
         }
 
     async def prepare_app(
             self,
-            app: DeployApp,
-            rev: DeployRev,
-            repo: DeployGitRepo,
+            spec: DeploySpec,
     ):
-        app_tag = DeployAppTag(app, make_deploy_tag(rev))
-        app_dir = os.path.join(self._dir, app, app_tag.tag)
+        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev, spec.key()))
+        app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
 
         #
 
         await self._git.checkout(
-            DeployGitSpec(
-                repo=repo,
-                rev=rev,
-            ),
+            spec.repo,
+            spec.rev,
             app_dir,
         )
 

ominfra/manage/deploy/atomics.py

@@ -0,0 +1,207 @@
+# ruff: noqa: UP006 UP007
+import abc
+import os
+import shutil
+import tempfile
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.strings import attr_repr
+
+
+DeployAtomicPathSwapKind = ta.Literal['dir', 'file']
+DeployAtomicPathSwapState = ta.Literal['open', 'committed', 'aborted']  # ta.TypeAlias
+
+
+##
+
+
+class DeployAtomicPathSwap(abc.ABC):
+    def __init__(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            *,
+            auto_commit: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self._kind = kind
+        self._dst_path = dst_path
+        self._auto_commit = auto_commit
+
+        self._state: DeployAtomicPathSwapState = 'open'
+
+    def __repr__(self) -> str:
+        return attr_repr(self, 'kind', 'dst_path', 'tmp_path')
+
+    @property
+    def kind(self) -> DeployAtomicPathSwapKind:
+        return self._kind
+
+    @property
+    def dst_path(self) -> str:
+        return self._dst_path
+
+    @property
+    @abc.abstractmethod
+    def tmp_path(self) -> str:
+        raise NotImplementedError
+
+    #
+
+    @property
+    def state(self) -> DeployAtomicPathSwapState:
+        return self._state
+
+    def _check_state(self, *states: DeployAtomicPathSwapState) -> None:
+        if self._state not in states:
+            raise RuntimeError(f'Atomic path swap not in correct state: {self._state}, {states}')
+
+    #
+
+    @abc.abstractmethod
+    def _commit(self) -> None:
+        raise NotImplementedError
+
+    def commit(self) -> None:
+        if self._state == 'committed':
+            return
+        self._check_state('open')
+        try:
+            self._commit()
+        except Exception:  # noqa
+            self._abort()
+            raise
+        else:
+            self._state = 'committed'
+
+    #
+
+    @abc.abstractmethod
+    def _abort(self) -> None:
+        raise NotImplementedError
+
+    def abort(self) -> None:
+        if self._state == 'aborted':
+            return
+        self._abort()
+        self._state = 'aborted'
+
+    #
+
+    def __enter__(self) -> 'DeployAtomicPathSwap':
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if (
+                exc_type is None and
+                self._auto_commit and
+                self._state == 'open'
+        ):
+            self.commit()
+        else:
+            self.abort()
+
+
+#
+
+
+class DeployAtomicPathSwapping(abc.ABC):
+    @abc.abstractmethod
+    def begin_atomic_path_swap(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            *,
+            name_hint: ta.Optional[str] = None,
+            make_dirs: bool = False,
+            **kwargs: ta.Any,
+    ) -> DeployAtomicPathSwap:
+        raise NotImplementedError
+
+
+##
+
+
+class OsRenameDeployAtomicPathSwap(DeployAtomicPathSwap):
+    def __init__(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            tmp_path: str,
+            **kwargs: ta.Any,
+    ) -> None:
+        if kind == 'dir':
+            check.state(os.path.isdir(tmp_path))
+        elif kind == 'file':
+            check.state(os.path.isfile(tmp_path))
+        else:
+            raise TypeError(kind)
+
+        super().__init__(
+            kind,
+            dst_path,
+            **kwargs,
+        )
+
+        self._tmp_path = tmp_path
+
+    @property
+    def tmp_path(self) -> str:
+        return self._tmp_path
+
+    def _commit(self) -> None:
+        os.rename(self._tmp_path, self._dst_path)
+
+    def _abort(self) -> None:
+        shutil.rmtree(self._tmp_path, ignore_errors=True)
+
+
+class TempDirDeployAtomicPathSwapping(DeployAtomicPathSwapping):
+    def __init__(
+            self,
+            *,
+            temp_dir: ta.Optional[str] = None,
+            root_dir: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+
+        if root_dir is not None:
+            root_dir = os.path.abspath(root_dir)
+        self._root_dir = root_dir
+        self._temp_dir = temp_dir
+
+    def begin_atomic_path_swap(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            *,
+            name_hint: ta.Optional[str] = None,
+            make_dirs: bool = False,
+            **kwargs: ta.Any,
+    ) -> DeployAtomicPathSwap:
+        dst_path = os.path.abspath(dst_path)
+        if self._root_dir is not None and not dst_path.startswith(check.non_empty_str(self._root_dir)):
+            raise RuntimeError(f'Atomic path swap dst must be in root dir: {dst_path}, {self._root_dir}')
+
+        dst_dir = os.path.dirname(dst_path)
+        if make_dirs:
+            os.makedirs(dst_dir, exist_ok=True)
+        if not os.path.isdir(dst_dir):
+            raise RuntimeError(f'Atomic path swap dst dir does not exist: {dst_dir}')
+
+        if kind == 'dir':
+            tmp_path = tempfile.mkdtemp(prefix=name_hint, dir=self._temp_dir)
+        elif kind == 'file':
+            fd, tmp_path = tempfile.mkstemp(prefix=name_hint, dir=self._temp_dir)
+            os.close(fd)
+        else:
+            raise TypeError(kind)
+
+        return OsRenameDeployAtomicPathSwap(
+            kind,
+            dst_path,
+            tmp_path,
+            **kwargs,
+        )

ominfra/manage/deploy/config.py

@@ -3,6 +3,9 @@ import dataclasses as dc
 import typing as ta
 
 
+##
+
+
 @dc.dataclass(frozen=True)
 class DeployConfig:
     deploy_home: ta.Optional[str] = None

ominfra/manage/deploy/git.py

@@ -8,17 +8,17 @@ git/github.com/wrmsr/omlish <- bootstrap repo
 
 github.com/wrmsr/omlish@rev
 """
-import dataclasses as dc
 import functools
 import os.path
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.cached import async_cached_nullary
 from omlish.lite.check import check
 
-from .paths import DeployPath
-from .paths import DeployPathOwner
+from .atomics import DeployAtomicPathSwapping
+from .paths import SingleDirDeployPathOwner
+from .specs import DeployGitRepo
 from .types import DeployHome
 from .types import DeployRev
 
@@ -26,44 +26,22 @@ from .types import DeployRev
 ##
 
 
-@dc.dataclass(frozen=True)
-class DeployGitRepo:
-    host: ta.Optional[str] = None
-    username: ta.Optional[str] = None
-    path: ta.Optional[str] = None
-
-    def __post_init__(self) -> None:
-        check.not_in('..', check.non_empty_str(self.host))
-        check.not_in('.', check.non_empty_str(self.path))
-
-
-@dc.dataclass(frozen=True)
-class DeployGitSpec:
-    repo: DeployGitRepo
-    rev: DeployRev
-
-
-##
-
-
-class DeployGitManager(DeployPathOwner):
+class DeployGitManager(SingleDirDeployPathOwner):
     def __init__(
             self,
             *,
-            deploy_home: DeployHome,
+            deploy_home: ta.Optional[DeployHome] = None,
+            atomics: DeployAtomicPathSwapping,
     ) -> None:
-        super().__init__()
+        super().__init__(
+            owned_dir='git',
+            deploy_home=deploy_home,
+        )
 
-        self._deploy_home = deploy_home
-        self._dir = os.path.join(deploy_home, 'git')
+        self._atomics = atomics
 
         self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
 
-    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
-        return {
-            DeployPath.parse('git'),
-        }
-
     class RepoDir:
         def __init__(
                 self,
@@ -75,7 +53,7 @@ class DeployGitManager(DeployPathOwner):
             self._git = git
             self._repo = repo
             self._dir = os.path.join(
-                self._git._dir,  # noqa
+                self._git._make_dir(),  # noqa
                 check.non_empty_str(repo.host),
                 check.non_empty_str(repo.path),
             )
@@ -112,18 +90,20 @@ class DeployGitManager(DeployPathOwner):
 
         async def checkout(self, rev: DeployRev, dst_dir: str) -> None:
             check.state(not os.path.exists(dst_dir))
+            with self._git._atomics.begin_atomic_path_swap(  # noqa
+                    'dir',
+                    dst_dir,
+                    auto_commit=True,
+                    make_dirs=True,
+            ) as dst_swap:
+                await self.fetch(rev)
 
-            await self.fetch(rev)
-
-            # FIXME: temp dir swap
-            os.makedirs(dst_dir)
-
-            dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_dir)
-            await dst_call('git', 'init')
+                dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
+                await dst_call('git', 'init')
 
-            await dst_call('git', 'remote', 'add', 'local', self._dir)
-            await dst_call('git', 'fetch', '--depth=1', 'local', rev)
-            await dst_call('git', 'checkout', rev)
+                await dst_call('git', 'remote', 'add', 'local', self._dir)
+                await dst_call('git', 'fetch', '--depth=1', 'local', rev)
+                await dst_call('git', 'checkout', rev)
 
     def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
         try:
@@ -132,5 +112,5 @@ class DeployGitManager(DeployPathOwner):
             repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
             return repo_dir
 
-    async def checkout(self, spec: DeployGitSpec, dst_dir: str) -> None:
-        await self.get_repo_dir(spec.repo).checkout(spec.rev, dst_dir)
+    async def checkout(self, repo: DeployGitRepo, rev: DeployRev, dst_dir: str) -> None:
+        await self.get_repo_dir(repo).checkout(rev, dst_dir)

ominfra/manage/deploy/inject.py

@@ -8,12 +8,14 @@ from omlish.lite.inject import inj
 
 from ..commands.inject import bind_command
 from .apps import DeployAppManager
+from .atomics import DeployAtomicPathSwapping
 from .commands import DeployCommand
 from .commands import DeployCommandExecutor
 from .config import DeployConfig
 from .git import DeployGitManager
 from .interp import InterpCommand
 from .interp import InterpCommandExecutor
+from .tmp import DeployTmpManager
 from .types import DeployHome
 from .venvs import DeployVenvManager
 
@@ -25,10 +27,19 @@ def bind_deploy(
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(deploy_config),
 
+        #
+
         inj.bind(DeployAppManager, singleton=True),
+
         inj.bind(DeployGitManager, singleton=True),
+
+        inj.bind(DeployTmpManager, singleton=True),
+        inj.bind(DeployAtomicPathSwapping, to_key=DeployTmpManager),
+
         inj.bind(DeployVenvManager, singleton=True),
 
+        #
+
         bind_command(DeployCommand, DeployCommandExecutor),
         bind_command(InterpCommand, InterpCommandExecutor),
     ]