ominfra 0.0.0.dev158__py3-none-any.whl → 0.0.0.dev159__py3-none-any.whl

ominfra/manage/deploy/apps.py

@@ -12,6 +12,7 @@ from .paths import DeployPathOwner
 from .specs import DeploySpec
 from .types import DeployAppTag
 from .types import DeployHome
+from .types import DeployKey
 from .types import DeployRev
 from .types import DeployTag
 from .venvs import DeployVenvManager
@@ -19,13 +20,15 @@ from .venvs import DeployVenvManager
 
 def make_deploy_tag(
         rev: DeployRev,
-        now: ta.Optional[datetime.datetime] = None,
+        key: DeployKey,
+        *,
+        utcnow: ta.Optional[datetime.datetime] = None,
 ) -> DeployTag:
-    if now is None:
-        now = datetime.datetime.utcnow()  # noqa
-    now_fmt = '%Y%m%dT%H%M%S'
-    now_str = now.strftime(now_fmt)
-    return DeployTag('-'.join([now_str, rev]))
+    if utcnow is None:
+        utcnow = datetime.datetime.now(tz=datetime.timezone.utc)  # noqa
+    now_fmt = '%Y%m%dT%H%M%SZ'
+    now_str = utcnow.strftime(now_fmt)
+    return DeployTag('-'.join([now_str, rev, key]))
 
 
 class DeployAppManager(DeployPathOwner):
@@ -46,7 +49,7 @@ class DeployAppManager(DeployPathOwner):
     def _dir(self) -> str:
         return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
 
-    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
         return {
             DeployPath.parse('apps/@app/@tag'),
         }
@@ -55,7 +58,7 @@ class DeployAppManager(DeployPathOwner):
             self,
             spec: DeploySpec,
     ):
-        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev))
+        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev, spec.key()))
         app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
 
         #

ominfra/manage/deploy/atomics.py

@@ -0,0 +1,207 @@
+# ruff: noqa: UP006 UP007
+import abc
+import os
+import shutil
+import tempfile
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.strings import attr_repr
+
+
+DeployAtomicPathSwapKind = ta.Literal['dir', 'file']
+DeployAtomicPathSwapState = ta.Literal['open', 'committed', 'aborted']  # ta.TypeAlias
+
+
+##
+
+
+class DeployAtomicPathSwap(abc.ABC):
+    def __init__(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            *,
+            auto_commit: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self._kind = kind
+        self._dst_path = dst_path
+        self._auto_commit = auto_commit
+
+        self._state: DeployAtomicPathSwapState = 'open'
+
+    def __repr__(self) -> str:
+        return attr_repr(self, 'kind', 'dst_path', 'tmp_path')
+
+    @property
+    def kind(self) -> DeployAtomicPathSwapKind:
+        return self._kind
+
+    @property
+    def dst_path(self) -> str:
+        return self._dst_path
+
+    @property
+    @abc.abstractmethod
+    def tmp_path(self) -> str:
+        raise NotImplementedError
+
+    #
+
+    @property
+    def state(self) -> DeployAtomicPathSwapState:
+        return self._state
+
+    def _check_state(self, *states: DeployAtomicPathSwapState) -> None:
+        if self._state not in states:
+            raise RuntimeError(f'Atomic path swap not in correct state: {self._state}, {states}')
+
+    #
+
+    @abc.abstractmethod
+    def _commit(self) -> None:
+        raise NotImplementedError
+
+    def commit(self) -> None:
+        if self._state == 'committed':
+            return
+        self._check_state('open')
+        try:
+            self._commit()
+        except Exception:  # noqa
+            self._abort()
+            raise
+        else:
+            self._state = 'committed'
+
+    #
+
+    @abc.abstractmethod
+    def _abort(self) -> None:
+        raise NotImplementedError
+
+    def abort(self) -> None:
+        if self._state == 'aborted':
+            return
+        self._abort()
+        self._state = 'aborted'
+
+    #
+
+    def __enter__(self) -> 'DeployAtomicPathSwap':
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if (
+                exc_type is None and
+                self._auto_commit and
+                self._state == 'open'
+        ):
+            self.commit()
+        else:
+            self.abort()
+
+
+#
+
+
+class DeployAtomicPathSwapping(abc.ABC):
+    @abc.abstractmethod
+    def begin_atomic_path_swap(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            *,
+            name_hint: ta.Optional[str] = None,
+            make_dirs: bool = False,
+            **kwargs: ta.Any,
+    ) -> DeployAtomicPathSwap:
+        raise NotImplementedError
+
+
+##
+
+
+class OsRenameDeployAtomicPathSwap(DeployAtomicPathSwap):
+    def __init__(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            tmp_path: str,
+            **kwargs: ta.Any,
+    ) -> None:
+        if kind == 'dir':
+            check.state(os.path.isdir(tmp_path))
+        elif kind == 'file':
+            check.state(os.path.isfile(tmp_path))
+        else:
+            raise TypeError(kind)
+
+        super().__init__(
+            kind,
+            dst_path,
+            **kwargs,
+        )
+
+        self._tmp_path = tmp_path
+
+    @property
+    def tmp_path(self) -> str:
+        return self._tmp_path
+
+    def _commit(self) -> None:
+        os.rename(self._tmp_path, self._dst_path)
+
+    def _abort(self) -> None:
+        shutil.rmtree(self._tmp_path, ignore_errors=True)
+
+
+class TempDirDeployAtomicPathSwapping(DeployAtomicPathSwapping):
+    def __init__(
+            self,
+            *,
+            temp_dir: ta.Optional[str] = None,
+            root_dir: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+
+        if root_dir is not None:
+            root_dir = os.path.abspath(root_dir)
+        self._root_dir = root_dir
+        self._temp_dir = temp_dir
+
+    def begin_atomic_path_swap(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            *,
+            name_hint: ta.Optional[str] = None,
+            make_dirs: bool = False,
+            **kwargs: ta.Any,
+    ) -> DeployAtomicPathSwap:
+        dst_path = os.path.abspath(dst_path)
+        if self._root_dir is not None and not dst_path.startswith(check.non_empty_str(self._root_dir)):
+            raise RuntimeError(f'Atomic path swap dst must be in root dir: {dst_path}, {self._root_dir}')
+
+        dst_dir = os.path.dirname(dst_path)
+        if make_dirs:
+            os.makedirs(dst_dir, exist_ok=True)
+        if not os.path.isdir(dst_dir):
+            raise RuntimeError(f'Atomic path swap dst dir does not exist: {dst_dir}')
+
+        if kind == 'dir':
+            tmp_path = tempfile.mkdtemp(prefix=name_hint, dir=self._temp_dir)
+        elif kind == 'file':
+            fd, tmp_path = tempfile.mkstemp(prefix=name_hint, dir=self._temp_dir)
+            os.close(fd)
+        else:
+            raise TypeError(kind)
+
+        return OsRenameDeployAtomicPathSwap(
+            kind,
+            dst_path,
+            tmp_path,
+            **kwargs,
+        )

ominfra/manage/deploy/git.py

@@ -14,11 +14,10 @@ import typing as ta
 
 from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.cached import async_cached_nullary
-from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 
-from .paths import DeployPath
-from .paths import DeployPathOwner
+from .atomics import DeployAtomicPathSwapping
+from .paths import SingleDirDeployPathOwner
 from .specs import DeployGitRepo
 from .types import DeployHome
 from .types import DeployRev
@@ -27,27 +26,22 @@ from .types import DeployRev
 ##
 
 
-class DeployGitManager(DeployPathOwner):
+class DeployGitManager(SingleDirDeployPathOwner):
     def __init__(
             self,
             *,
             deploy_home: ta.Optional[DeployHome] = None,
+            atomics: DeployAtomicPathSwapping,
     ) -> None:
-        super().__init__()
+        super().__init__(
+            owned_dir='git',
+            deploy_home=deploy_home,
+        )
 
-        self._deploy_home = deploy_home
+        self._atomics = atomics
 
         self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
 
-    @cached_nullary
-    def _dir(self) -> str:
-        return os.path.join(check.non_empty_str(self._deploy_home), 'git')
-
-    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
-        return {
-            DeployPath.parse('git'),
-        }
-
     class RepoDir:
         def __init__(
                 self,
@@ -59,7 +53,7 @@ class DeployGitManager(DeployPathOwner):
             self._git = git
             self._repo = repo
             self._dir = os.path.join(
-                self._git._dir(),  # noqa
+                self._git._make_dir(),  # noqa
                 check.non_empty_str(repo.host),
                 check.non_empty_str(repo.path),
             )
@@ -96,18 +90,20 @@ class DeployGitManager(DeployPathOwner):
 
         async def checkout(self, rev: DeployRev, dst_dir: str) -> None:
             check.state(not os.path.exists(dst_dir))
-
-            await self.fetch(rev)
-
-            # FIXME: temp dir swap
-            os.makedirs(dst_dir)
-
-            dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_dir)
-            await dst_call('git', 'init')
-
-            await dst_call('git', 'remote', 'add', 'local', self._dir)
-            await dst_call('git', 'fetch', '--depth=1', 'local', rev)
-            await dst_call('git', 'checkout', rev)
+            with self._git._atomics.begin_atomic_path_swap(  # noqa
+                    'dir',
+                    dst_dir,
+                    auto_commit=True,
+                    make_dirs=True,
+            ) as dst_swap:
+                await self.fetch(rev)
+
+                dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
+                await dst_call('git', 'init')
+
+                await dst_call('git', 'remote', 'add', 'local', self._dir)
+                await dst_call('git', 'fetch', '--depth=1', 'local', rev)
+                await dst_call('git', 'checkout', rev)
 
     def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
         try:

ominfra/manage/deploy/inject.py

@@ -8,12 +8,14 @@ from omlish.lite.inject import inj
 
 from ..commands.inject import bind_command
 from .apps import DeployAppManager
+from .atomics import DeployAtomicPathSwapping
 from .commands import DeployCommand
 from .commands import DeployCommandExecutor
 from .config import DeployConfig
 from .git import DeployGitManager
 from .interp import InterpCommand
 from .interp import InterpCommandExecutor
+from .tmp import DeployTmpManager
 from .types import DeployHome
 from .venvs import DeployVenvManager
 
@@ -25,10 +27,19 @@ def bind_deploy(
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(deploy_config),
 
+        #
+
         inj.bind(DeployAppManager, singleton=True),
+
         inj.bind(DeployGitManager, singleton=True),
+
+        inj.bind(DeployTmpManager, singleton=True),
+        inj.bind(DeployAtomicPathSwapping, to_key=DeployTmpManager),
+
         inj.bind(DeployVenvManager, singleton=True),
 
+        #
+
         bind_command(DeployCommand, DeployCommandExecutor),
         bind_command(InterpCommand, InterpCommandExecutor),
     ]

ominfra/manage/deploy/paths.py

@@ -14,6 +14,8 @@
   /venv
     /<appplaceholder>
 
+  /tmp
+
 ?
   /logs
     /wrmsr--omlish--<placeholder>
@@ -39,8 +41,11 @@ import dataclasses as dc
 import os.path
 import typing as ta
 
+from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 
+from .types import DeployHome
+
 
 DeployPathKind = ta.Literal['dir', 'file']  # ta.TypeAlias
 DeployPathPlaceholder = ta.Literal['app', 'tag']  # ta.TypeAlias
@@ -181,6 +186,8 @@ class DeployPath:
     parts: ta.Sequence[DeployPathPart]
 
     def __post_init__(self) -> None:
+        hash(self)
+
         check.not_empty(self.parts)
         for p in self.parts[:-1]:
             check.equal(p.kind, 'dir')
@@ -215,10 +222,10 @@ class DeployPath:
         else:
             tail_parse = FileDeployPathPart.parse
         ps = check.non_empty_str(s).split('/')
-        return cls([
+        return cls((
             *([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
             tail_parse(ps[-1]),
-        ])
+        ))
 
 
 ##
@@ -226,5 +233,36 @@ class DeployPath:
 
 class DeployPathOwner(abc.ABC):
     @abc.abstractmethod
-    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
         raise NotImplementedError
+
+
+class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
+    def __init__(
+            self,
+            *args: ta.Any,
+            owned_dir: str,
+            deploy_home: ta.Optional[DeployHome],
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+
+        check.not_in('/', owned_dir)
+        self._owned_dir: str = check.non_empty_str(owned_dir)
+
+        self._deploy_home = deploy_home
+
+        self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
+
+    @cached_nullary
+    def _dir(self) -> str:
+        return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
+
+    @cached_nullary
+    def _make_dir(self) -> str:
+        if not os.path.isdir(d := self._dir()):
+            os.makedirs(d, exist_ok=True)
+        return d
+
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return self._owned_deploy_paths

ominfra/manage/deploy/specs.py

@@ -1,10 +1,13 @@
 # ruff: noqa: UP006 UP007
 import dataclasses as dc
+import hashlib
 import typing as ta
 
+from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 
 from .types import DeployApp
+from .types import DeployKey
 from .types import DeployRev
 
 
@@ -30,3 +33,10 @@ class DeploySpec:
     app: DeployApp
     repo: DeployGitRepo
     rev: DeployRev
+
+    def __post_init__(self) -> None:
+        hash(self)
+
+    @cached_nullary
+    def key(self) -> DeployKey:
+        return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])

ominfra/manage/deploy/tmp.py

@@ -0,0 +1,46 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+
+from .atomics import DeployAtomicPathSwap
+from .atomics import DeployAtomicPathSwapKind
+from .atomics import DeployAtomicPathSwapping
+from .atomics import TempDirDeployAtomicPathSwapping
+from .paths import SingleDirDeployPathOwner
+from .types import DeployHome
+
+
+class DeployTmpManager(
+    SingleDirDeployPathOwner,
+    DeployAtomicPathSwapping,
+):
+    def __init__(
+            self,
+            *,
+            deploy_home: ta.Optional[DeployHome] = None,
+    ) -> None:
+        super().__init__(
+            owned_dir='tmp',
+            deploy_home=deploy_home,
+        )
+
+    @cached_nullary
+    def _swapping(self) -> DeployAtomicPathSwapping:
+        return TempDirDeployAtomicPathSwapping(
+            temp_dir=self._make_dir(),
+            root_dir=check.non_empty_str(self._deploy_home),
+        )
+
+    def begin_atomic_path_swap(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            **kwargs: ta.Any,
+    ) -> DeployAtomicPathSwap:
+        return self._swapping().begin_atomic_path_swap(
+            kind,
+            dst_path,
+            **kwargs,
+        )

ominfra/manage/deploy/types.py

@@ -6,6 +6,7 @@ DeployHome = ta.NewType('DeployHome', str)
 DeployApp = ta.NewType('DeployApp', str)
 DeployTag = ta.NewType('DeployTag', str)
 DeployRev = ta.NewType('DeployRev', str)
+DeployKey = ta.NewType('DeployKey', str)
 
 
 class DeployAppTag(ta.NamedTuple):

ominfra/manage/deploy/venvs.py

@@ -11,6 +11,7 @@ from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 
+from .atomics import DeployAtomicPathSwapping
 from .paths import DeployPath
 from .paths import DeployPathOwner
 from .types import DeployAppTag
@@ -22,16 +23,18 @@ class DeployVenvManager(DeployPathOwner):
             self,
             *,
             deploy_home: ta.Optional[DeployHome] = None,
+            atomics: DeployAtomicPathSwapping,
     ) -> None:
         super().__init__()
 
         self._deploy_home = deploy_home
+        self._atomics = atomics
 
     @cached_nullary
     def _dir(self) -> str:
         return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
 
-    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
         return {
             DeployPath.parse('venvs/@app/@tag/'),
         }
@@ -45,6 +48,8 @@ class DeployVenvManager(DeployPathOwner):
     ) -> None:
         sys_exe = 'python3'
 
+        # !! NOTE: (most) venvs cannot be relocated, so an atomic swap can't be used. it's up to the path manager to
+        # garbage collect orphaned dirs.
         await asyncio_subprocesses.check_call(sys_exe, '-m', 'venv', venv_dir)
 
         #

ominfra/scripts/journald2aws.py

@@ -2899,12 +2899,12 @@ def is_debugger_attached() -> bool:
     return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
 
 
-REQUIRED_PYTHON_VERSION = (3, 8)
+LITE_REQUIRED_PYTHON_VERSION = (3, 8)
 
 
-def check_runtime_version() -> None:
-    if sys.version_info < REQUIRED_PYTHON_VERSION:
-        raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}')  # noqa
+def check_lite_runtime_version() -> None:
+    if sys.version_info < LITE_REQUIRED_PYTHON_VERSION:
+        raise OSError(f'Requires python {LITE_REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}')  # noqa
 
 
 ########################################
@@ -3482,6 +3482,7 @@ TODO:
  - structured
  - prefixed
  - debug
+ - optional noisy? noisy will never be lite - some kinda configure_standard callback mechanism?
 """
 
 
@@ -3518,8 +3519,9 @@ class StandardLogFormatter(logging.Formatter):
 ##
 
 
-class StandardLogHandler(ProxyLogHandler):
-    pass
+class StandardConfiguredLogHandler(ProxyLogHandler):
+    def __init_subclass__(cls, **kwargs):
+        raise TypeError('This class serves only as a marker and should not be subclassed.')
 
 
 ##
@@ -3550,7 +3552,7 @@ def configure_standard_logging(
         target: ta.Optional[logging.Logger] = None,
         force: bool = False,
         handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
-) -> ta.Optional[StandardLogHandler]:
+) -> ta.Optional[StandardConfiguredLogHandler]:
     with _locking_logging_module_lock():
         if target is None:
             target = logging.root
@@ -3558,7 +3560,7 @@ def configure_standard_logging(
         #
 
         if not force:
-            if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
+            if any(isinstance(h, StandardConfiguredLogHandler) for h in list(target.handlers)):
                 return None
 
         #
@@ -3592,7 +3594,7 @@ def configure_standard_logging(
 
         #
 
-        return StandardLogHandler(handler)
+        return StandardConfiguredLogHandler(handler)
 
 
 ########################################