ominfra 0.0.0.dev157__py3-none-any.whl → 0.0.0.dev159__py3-none-any.whl

ominfra/manage/deploy/paths.py

@@ -3,22 +3,24 @@
 ~deploy
   deploy.pid (flock)
   /app
-    /<appspec> - shallow clone
+    /<appplaceholder> - shallow clone
   /conf
     /env
-      <appspec>.env
+      <appplaceholder>.env
     /nginx
-      <appspec>.conf
+      <appplaceholder>.conf
     /supervisor
-      <appspec>.conf
+      <appplaceholder>.conf
   /venv
-    /<appspec>
+    /<appplaceholder>
+
+  /tmp
 
 ?
   /logs
-    /wrmsr--omlish--<spec>
+    /wrmsr--omlish--<placeholder>
 
-spec = <name>--<rev>--<when>
+placeholder = <name>--<rev>--<when>
 
 ==
 
@@ -39,20 +41,23 @@ import dataclasses as dc
 import os.path
 import typing as ta
 
+from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 
+from .types import DeployHome
+
 
 DeployPathKind = ta.Literal['dir', 'file']  # ta.TypeAlias
-DeployPathSpec = ta.Literal['app', 'tag']  # ta.TypeAlias
+DeployPathPlaceholder = ta.Literal['app', 'tag']  # ta.TypeAlias
 
 
 ##
 
 
-DEPLOY_PATH_SPEC_PLACEHOLDER = '@'
-DEPLOY_PATH_SPEC_SEPARATORS = '-.'
+DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER = '@'
+DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
 
-DEPLOY_PATH_SPECS: ta.FrozenSet[str] = frozenset([
+DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
     'app',
     'tag',  # <rev>-<dt>
 ])
@@ -70,7 +75,7 @@ class DeployPathPart(abc.ABC):  # noqa
         raise NotImplementedError
 
     @abc.abstractmethod
-    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
         raise NotImplementedError
 
 
@@ -84,9 +89,9 @@ class DirDeployPathPart(DeployPathPart, abc.ABC):
 
     @classmethod
     def parse(cls, s: str) -> 'DirDeployPathPart':
-        if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
-            check.equal(s[0], DEPLOY_PATH_SPEC_PLACEHOLDER)
-            return SpecDirDeployPathPart(s[1:])
+        if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
+            check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
+            return PlaceholderDirDeployPathPart(s[1:])
         else:
             return ConstDirDeployPathPart(s)
 
@@ -98,13 +103,13 @@ class FileDeployPathPart(DeployPathPart, abc.ABC):
 
     @classmethod
     def parse(cls, s: str) -> 'FileDeployPathPart':
-        if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
-            check.equal(s[0], DEPLOY_PATH_SPEC_PLACEHOLDER)
-            if not any(c in s for c in DEPLOY_PATH_SPEC_SEPARATORS):
-                return SpecFileDeployPathPart(s[1:], '')
+        if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
+            check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
+            if not any(c in s for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS):
+                return PlaceholderFileDeployPathPart(s[1:], '')
             else:
-                p = min(f for c in DEPLOY_PATH_SPEC_SEPARATORS if (f := s.find(c)) > 0)
-                return SpecFileDeployPathPart(s[1:p], s[p:])
+                p = min(f for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS if (f := s.find(c)) > 0)
+                return PlaceholderFileDeployPathPart(s[1:p], s[p:])
         else:
             return ConstFileDeployPathPart(s)
 
@@ -119,9 +124,9 @@ class ConstDeployPathPart(DeployPathPart, abc.ABC):
     def __post_init__(self) -> None:
         check.non_empty_str(self.name)
         check.not_in('/', self.name)
-        check.not_in(DEPLOY_PATH_SPEC_PLACEHOLDER, self.name)
+        check.not_in(DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, self.name)
 
-    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
         return self.name
 
 
@@ -137,40 +142,40 @@ class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
 
 
 @dc.dataclass(frozen=True)
-class SpecDeployPathPart(DeployPathPart, abc.ABC):
-    spec: str  # DeployPathSpec
+class PlaceholderDeployPathPart(DeployPathPart, abc.ABC):
+    placeholder: str  # DeployPathPlaceholder
 
     def __post_init__(self) -> None:
-        check.non_empty_str(self.spec)
-        for c in [*DEPLOY_PATH_SPEC_SEPARATORS, DEPLOY_PATH_SPEC_PLACEHOLDER, '/']:
-            check.not_in(c, self.spec)
-        check.in_(self.spec, DEPLOY_PATH_SPECS)
-
-    def _render_spec(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
-        if specs is not None:
-            return specs[self.spec]  # type: ignore
+        check.non_empty_str(self.placeholder)
+        for c in [*DEPLOY_PATH_PLACEHOLDER_SEPARATORS, DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
+            check.not_in(c, self.placeholder)
+        check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
+
+    def _render_placeholder(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+        if placeholders is not None:
+            return placeholders[self.placeholder]  # type: ignore
         else:
-            return DEPLOY_PATH_SPEC_PLACEHOLDER + self.spec
+            return DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER + self.placeholder
 
 
 @dc.dataclass(frozen=True)
-class SpecDirDeployPathPart(SpecDeployPathPart, DirDeployPathPart):
-    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
-        return self._render_spec(specs)
+class PlaceholderDirDeployPathPart(PlaceholderDeployPathPart, DirDeployPathPart):
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+        return self._render_placeholder(placeholders)
 
 
 @dc.dataclass(frozen=True)
-class SpecFileDeployPathPart(SpecDeployPathPart, FileDeployPathPart):
+class PlaceholderFileDeployPathPart(PlaceholderDeployPathPart, FileDeployPathPart):
     suffix: str
 
     def __post_init__(self) -> None:
         super().__post_init__()
         if self.suffix:
-            for c in [DEPLOY_PATH_SPEC_PLACEHOLDER, '/']:
+            for c in [DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
                 check.not_in(c, self.suffix)
 
-    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
-        return self._render_spec(specs) + self.suffix
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+        return self._render_placeholder(placeholders) + self.suffix
 
 
 ##
@@ -181,28 +186,30 @@ class DeployPath:
     parts: ta.Sequence[DeployPathPart]
 
     def __post_init__(self) -> None:
+        hash(self)
+
         check.not_empty(self.parts)
         for p in self.parts[:-1]:
             check.equal(p.kind, 'dir')
 
         pd = {}
         for i, p in enumerate(self.parts):
-            if isinstance(p, SpecDeployPathPart):
-                if p.spec in pd:
-                    raise DeployPathError('Duplicate specs in path', self)
-                pd[p.spec] = i
+            if isinstance(p, PlaceholderDeployPathPart):
+                if p.placeholder in pd:
+                    raise DeployPathError('Duplicate placeholders in path', self)
+                pd[p.placeholder] = i
 
         if 'tag' in pd:
             if 'app' not in pd or pd['app'] >= pd['tag']:
-                raise DeployPathError('Tag spec in path without preceding app', self)
+                raise DeployPathError('Tag placeholder in path without preceding app', self)
 
     @property
     def kind(self) -> ta.Literal['file', 'dir']:
         return self.parts[-1].kind
 
-    def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
         return os.path.join(  # noqa
-            *[p.render(specs) for p in self.parts],
+            *[p.render(placeholders) for p in self.parts],
             *([''] if self.kind == 'dir' else []),
         )
 
@@ -215,10 +222,10 @@ class DeployPath:
         else:
             tail_parse = FileDeployPathPart.parse
         ps = check.non_empty_str(s).split('/')
-        return cls([
+        return cls((
             *([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
             tail_parse(ps[-1]),
-        ])
+        ))
 
 
 ##
@@ -226,5 +233,36 @@ class DeployPath:
 
 class DeployPathOwner(abc.ABC):
     @abc.abstractmethod
-    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
         raise NotImplementedError
+
+
+class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
+    def __init__(
+            self,
+            *args: ta.Any,
+            owned_dir: str,
+            deploy_home: ta.Optional[DeployHome],
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+
+        check.not_in('/', owned_dir)
+        self._owned_dir: str = check.non_empty_str(owned_dir)
+
+        self._deploy_home = deploy_home
+
+        self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
+
+    @cached_nullary
+    def _dir(self) -> str:
+        return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
+
+    @cached_nullary
+    def _make_dir(self) -> str:
+        if not os.path.isdir(d := self._dir()):
+            os.makedirs(d, exist_ok=True)
+        return d
+
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+        return self._owned_deploy_paths

ominfra/manage/deploy/specs.py

@@ -0,0 +1,42 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import hashlib
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+
+from .types import DeployApp
+from .types import DeployKey
+from .types import DeployRev
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployGitRepo:
+    host: ta.Optional[str] = None
+    username: ta.Optional[str] = None
+    path: ta.Optional[str] = None
+
+    def __post_init__(self) -> None:
+        check.not_in('..', check.non_empty_str(self.host))
+        check.not_in('.', check.non_empty_str(self.path))
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeploySpec:
+    app: DeployApp
+    repo: DeployGitRepo
+    rev: DeployRev
+
+    def __post_init__(self) -> None:
+        hash(self)
+
+    @cached_nullary
+    def key(self) -> DeployKey:
+        return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])

ominfra/manage/deploy/tmp.py

@@ -0,0 +1,46 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+
+from .atomics import DeployAtomicPathSwap
+from .atomics import DeployAtomicPathSwapKind
+from .atomics import DeployAtomicPathSwapping
+from .atomics import TempDirDeployAtomicPathSwapping
+from .paths import SingleDirDeployPathOwner
+from .types import DeployHome
+
+
+class DeployTmpManager(
+    SingleDirDeployPathOwner,
+    DeployAtomicPathSwapping,
+):
+    def __init__(
+            self,
+            *,
+            deploy_home: ta.Optional[DeployHome] = None,
+    ) -> None:
+        super().__init__(
+            owned_dir='tmp',
+            deploy_home=deploy_home,
+        )
+
+    @cached_nullary
+    def _swapping(self) -> DeployAtomicPathSwapping:
+        return TempDirDeployAtomicPathSwapping(
+            temp_dir=self._make_dir(),
+            root_dir=check.non_empty_str(self._deploy_home),
+        )
+
+    def begin_atomic_path_swap(
+            self,
+            kind: DeployAtomicPathSwapKind,
+            dst_path: str,
+            **kwargs: ta.Any,
+    ) -> DeployAtomicPathSwap:
+        return self._swapping().begin_atomic_path_swap(
+            kind,
+            dst_path,
+            **kwargs,
+        )

ominfra/manage/deploy/types.py

@@ -6,6 +6,7 @@ DeployHome = ta.NewType('DeployHome', str)
 DeployApp = ta.NewType('DeployApp', str)
 DeployTag = ta.NewType('DeployTag', str)
 DeployRev = ta.NewType('DeployRev', str)
+DeployKey = ta.NewType('DeployKey', str)
 
 
 class DeployAppTag(ta.NamedTuple):

ominfra/manage/deploy/venvs.py

@@ -7,8 +7,11 @@ TODO:
 import os.path
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
 
+from .atomics import DeployAtomicPathSwapping
 from .paths import DeployPath
 from .paths import DeployPathOwner
 from .types import DeployAppTag
@@ -19,14 +22,19 @@ class DeployVenvManager(DeployPathOwner):
     def __init__(
             self,
             *,
-            deploy_home: DeployHome,
+            deploy_home: ta.Optional[DeployHome] = None,
+            atomics: DeployAtomicPathSwapping,
     ) -> None:
         super().__init__()
 
         self._deploy_home = deploy_home
-        self._dir = os.path.join(deploy_home, 'venvs')
+        self._atomics = atomics
 
-    def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
+    @cached_nullary
+    def _dir(self) -> str:
+        return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
+
+    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
         return {
             DeployPath.parse('venvs/@app/@tag/'),
         }
@@ -40,6 +48,8 @@ class DeployVenvManager(DeployPathOwner):
     ) -> None:
         sys_exe = 'python3'
 
+        # !! NOTE: (most) venvs cannot be relocated, so an atomic swap can't be used. it's up to the path manager to
+        # garbage collect orphaned dirs.
         await asyncio_subprocesses.check_call(sys_exe, '-m', 'venv', venv_dir)
 
         #
@@ -61,6 +71,6 @@ class DeployVenvManager(DeployPathOwner):
 
     async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
         await self.setup_venv(
-            os.path.join(self._deploy_home, 'apps', app_tag.app, app_tag.tag),
-            os.path.join(self._deploy_home, 'venvs', app_tag.app, app_tag.tag),
+            os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
+            os.path.join(self._dir(), app_tag.app, app_tag.tag),
         )

ominfra/manage/remote/spawning.py

@@ -7,10 +7,10 @@ import shlex
 import subprocess
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
-from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
-from omlish.lite.subprocesses import SubprocessChannelOption
+from omlish.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
+from omlish.subprocesses import SubprocessChannelOption
 
 
 ##

ominfra/manage/system/packages.py

@@ -9,7 +9,7 @@ import json
 import os
 import typing as ta
 
-from omlish.lite.asyncio.subprocesses import asyncio_subprocesses
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 
 

ominfra/pyremote.py

@@ -162,7 +162,7 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
     # Get pid
     pid = os.getpid()
 
-    # Two copies of main src to be sent to parent
+    # Two copies of payload src to be sent to parent
     r0, w0 = os.pipe()
     r1, w1 = os.pipe()
 
@@ -201,17 +201,17 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
         # Write pid
         os.write(1, struct.pack('<Q', pid))
 
-        # Read main src from stdin
-        main_z_len = struct.unpack('<I', os.read(0, 4))[0]
-        if len(main_z := os.fdopen(0, 'rb').read(main_z_len)) != main_z_len:
+        # Read payload src from stdin
+        payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
+        if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
             raise EOFError
-        main_src = zlib.decompress(main_z)
+        payload_src = zlib.decompress(payload_z)
 
-        # Write both copies of main src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely fill
-        # and block and need to be drained by pyremote_bootstrap_finalize running in parent.
+        # Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
+        # fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
         for w in [w0, w1]:
             fp = os.fdopen(w, 'wb', 0)
-            fp.write(main_src)
+            fp.write(payload_src)
             fp.close()
 
         # Write second ack
@@ -275,7 +275,7 @@ class PyremotePayloadRuntime:
     input: ta.BinaryIO
     output: ta.BinaryIO
     context_name: str
-    main_src: str
+    payload_src: str
     options: PyremoteBootstrapOptions
     env_info: PyremoteEnvInfo
 
@@ -283,9 +283,9 @@ class PyremotePayloadRuntime:
 def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
     # If src file var is not present we need to do initial finalization
     if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
-        # Read second copy of main src
+        # Read second copy of payload src
         r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
-        main_src = r1.read().decode('utf-8')
+        payload_src = r1.read().decode('utf-8')
         r1.close()
 
         # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
@@ -303,7 +303,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
             # Write temp source file
             import tempfile
             tfd, tfn = tempfile.mkstemp('-pyremote.py')
-            os.write(tfd, main_src.encode('utf-8'))
+            os.write(tfd, payload_src.encode('utf-8'))
             os.close(tfd)
 
             # Set vars
@@ -322,7 +322,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
 
         # Read temp source file
         with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
-            main_src = sf.read()
+            payload_src = sf.read()
 
     # Restore vars
     sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
@@ -355,7 +355,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
         input=input,
         output=output,
         context_name=context_name,
-        main_src=main_src,
+        payload_src=payload_src,
         options=options,
         env_info=env_info,
     )
@@ -367,31 +367,31 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
 class PyremoteBootstrapDriver:
     def __init__(
             self,
-            main_src: ta.Union[str, ta.Sequence[str]],
+            payload_src: ta.Union[str, ta.Sequence[str]],
             options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
     ) -> None:
         super().__init__()
 
-        self._main_src = main_src
+        self._payload_src = payload_src
         self._options = options
 
-        self._prepared_main_src = self._prepare_main_src(main_src, options)
-        self._main_z = zlib.compress(self._prepared_main_src.encode('utf-8'))
+        self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
+        self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
 
         self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8')  # noqa
     #
 
     @classmethod
-    def _prepare_main_src(
+    def _prepare_payload_src(
             cls,
-            main_src: ta.Union[str, ta.Sequence[str]],
+            payload_src: ta.Union[str, ta.Sequence[str]],
             options: PyremoteBootstrapOptions,
     ) -> str:
         parts: ta.List[str]
-        if isinstance(main_src, str):
-            parts = [main_src]
+        if isinstance(payload_src, str):
+            parts = [payload_src]
         else:
-            parts = list(main_src)
+            parts = list(payload_src)
 
         if (mn := options.main_name_override) is not None:
             parts.insert(0, f'__name__ = {mn!r}')
@@ -427,9 +427,9 @@ class PyremoteBootstrapDriver:
         d = yield from self._read(8)
         pid = struct.unpack('<Q', d)[0]
 
-        # Write main src
-        yield from self._write(struct.pack('<I', len(self._main_z)))
-        yield from self._write(self._main_z)
+        # Write payload src
+        yield from self._write(struct.pack('<I', len(self._payload_z)))
+        yield from self._write(self._payload_z)
 
         # Read second ack (after writing src copies)
         yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)