ominfra 0.0.0.dev159__py3-none-any.whl → 0.0.0.dev161__py3-none-any.whl

ominfra/scripts/manage.py

@@ -100,10 +100,6 @@ CallableVersionOperator = ta.Callable[['Version', str], bool]
 CommandT = ta.TypeVar('CommandT', bound='Command')
 CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
 
-# deploy/atomics.py
-DeployAtomicPathSwapKind = ta.Literal['dir', 'file']
-DeployAtomicPathSwapState = ta.Literal['open', 'committed', 'aborted']  # ta.TypeAlias
-
 # deploy/paths.py
 DeployPathKind = ta.Literal['dir', 'file']  # ta.TypeAlias
 DeployPathPlaceholder = ta.Literal['app', 'tag']  # ta.TypeAlias
@@ -121,6 +117,10 @@ InjectorProviderFn = ta.Callable[['Injector'], ta.Any]
 InjectorProviderFnMap = ta.Mapping['InjectorKey', 'InjectorProviderFn']
 InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
 
+# ../../omlish/os/atomics.py
+AtomicPathSwapKind = ta.Literal['dir', 'file']
+AtomicPathSwapState = ta.Literal['open', 'committed', 'aborted']  # ta.TypeAlias
+
 # ../configs.py
 ConfigMapping = ta.Mapping[str, ta.Any]
 
@@ -2692,6 +2692,10 @@ def is_new_type(spec: ta.Any) -> bool:
         return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1]  # type: ignore  # noqa
 
 
+def get_new_type_supertype(spec: ta.Any) -> ta.Any:
+    return spec.__supertype__
+
+
 def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
     seen = set()
     todo = list(reversed(cls.__subclasses__()))
@@ -4060,242 +4064,15 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
     return CommandNameMap(dct)
 
 
-########################################
-# ../deploy/atomics.py
-
-
-##
-
-
-class DeployAtomicPathSwap(abc.ABC):
-    def __init__(
-            self,
-            kind: DeployAtomicPathSwapKind,
-            dst_path: str,
-            *,
-            auto_commit: bool = False,
-    ) -> None:
-        super().__init__()
-
-        self._kind = kind
-        self._dst_path = dst_path
-        self._auto_commit = auto_commit
-
-        self._state: DeployAtomicPathSwapState = 'open'
-
-    def __repr__(self) -> str:
-        return attr_repr(self, 'kind', 'dst_path', 'tmp_path')
-
-    @property
-    def kind(self) -> DeployAtomicPathSwapKind:
-        return self._kind
-
-    @property
-    def dst_path(self) -> str:
-        return self._dst_path
-
-    @property
-    @abc.abstractmethod
-    def tmp_path(self) -> str:
-        raise NotImplementedError
-
-    #
-
-    @property
-    def state(self) -> DeployAtomicPathSwapState:
-        return self._state
-
-    def _check_state(self, *states: DeployAtomicPathSwapState) -> None:
-        if self._state not in states:
-            raise RuntimeError(f'Atomic path swap not in correct state: {self._state}, {states}')
-
-    #
-
-    @abc.abstractmethod
-    def _commit(self) -> None:
-        raise NotImplementedError
-
-    def commit(self) -> None:
-        if self._state == 'committed':
-            return
-        self._check_state('open')
-        try:
-            self._commit()
-        except Exception:  # noqa
-            self._abort()
-            raise
-        else:
-            self._state = 'committed'
-
-    #
-
-    @abc.abstractmethod
-    def _abort(self) -> None:
-        raise NotImplementedError
-
-    def abort(self) -> None:
-        if self._state == 'aborted':
-            return
-        self._abort()
-        self._state = 'aborted'
-
-    #
-
-    def __enter__(self) -> 'DeployAtomicPathSwap':
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if (
-                exc_type is None and
-                self._auto_commit and
-                self._state == 'open'
-        ):
-            self.commit()
-        else:
-            self.abort()
-
-
-#
-
-
-class DeployAtomicPathSwapping(abc.ABC):
-    @abc.abstractmethod
-    def begin_atomic_path_swap(
-            self,
-            kind: DeployAtomicPathSwapKind,
-            dst_path: str,
-            *,
-            name_hint: ta.Optional[str] = None,
-            make_dirs: bool = False,
-            **kwargs: ta.Any,
-    ) -> DeployAtomicPathSwap:
-        raise NotImplementedError
-
-
-##
-
-
-class OsRenameDeployAtomicPathSwap(DeployAtomicPathSwap):
-    def __init__(
-            self,
-            kind: DeployAtomicPathSwapKind,
-            dst_path: str,
-            tmp_path: str,
-            **kwargs: ta.Any,
-    ) -> None:
-        if kind == 'dir':
-            check.state(os.path.isdir(tmp_path))
-        elif kind == 'file':
-            check.state(os.path.isfile(tmp_path))
-        else:
-            raise TypeError(kind)
-
-        super().__init__(
-            kind,
-            dst_path,
-            **kwargs,
-        )
-
-        self._tmp_path = tmp_path
-
-    @property
-    def tmp_path(self) -> str:
-        return self._tmp_path
-
-    def _commit(self) -> None:
-        os.rename(self._tmp_path, self._dst_path)
-
-    def _abort(self) -> None:
-        shutil.rmtree(self._tmp_path, ignore_errors=True)
-
-
-class TempDirDeployAtomicPathSwapping(DeployAtomicPathSwapping):
-    def __init__(
-            self,
-            *,
-            temp_dir: ta.Optional[str] = None,
-            root_dir: ta.Optional[str] = None,
-    ) -> None:
-        super().__init__()
-
-        if root_dir is not None:
-            root_dir = os.path.abspath(root_dir)
-        self._root_dir = root_dir
-        self._temp_dir = temp_dir
-
-    def begin_atomic_path_swap(
-            self,
-            kind: DeployAtomicPathSwapKind,
-            dst_path: str,
-            *,
-            name_hint: ta.Optional[str] = None,
-            make_dirs: bool = False,
-            **kwargs: ta.Any,
-    ) -> DeployAtomicPathSwap:
-        dst_path = os.path.abspath(dst_path)
-        if self._root_dir is not None and not dst_path.startswith(check.non_empty_str(self._root_dir)):
-            raise RuntimeError(f'Atomic path swap dst must be in root dir: {dst_path}, {self._root_dir}')
-
-        dst_dir = os.path.dirname(dst_path)
-        if make_dirs:
-            os.makedirs(dst_dir, exist_ok=True)
-        if not os.path.isdir(dst_dir):
-            raise RuntimeError(f'Atomic path swap dst dir does not exist: {dst_dir}')
-
-        if kind == 'dir':
-            tmp_path = tempfile.mkdtemp(prefix=name_hint, dir=self._temp_dir)
-        elif kind == 'file':
-            fd, tmp_path = tempfile.mkstemp(prefix=name_hint, dir=self._temp_dir)
-            os.close(fd)
-        else:
-            raise TypeError(kind)
-
-        return OsRenameDeployAtomicPathSwap(
-            kind,
-            dst_path,
-            tmp_path,
-            **kwargs,
-        )
-
-
 ########################################
 # ../deploy/paths.py
 """
-~deploy
-  deploy.pid (flock)
-  /app
-    /<appplaceholder> - shallow clone
-  /conf
-    /env
-      <appplaceholder>.env
-    /nginx
-      <appplaceholder>.conf
-    /supervisor
-      <appplaceholder>.conf
-  /venv
-    /<appplaceholder>
-
-  /tmp
-
-?
-  /logs
-    /wrmsr--omlish--<placeholder>
-
-placeholder = <name>--<rev>--<when>
-
-==
-
-for dn in [
-    'app',
-    'conf',
-    'conf/env',
-    'conf/nginx',
-    'conf/supervisor',
-    'venv',
-]:
-
-==
-
+TODO:
+ - run/pidfile
+ - logs/...
+ - current symlink
+ - conf/{nginx,supervisor}
+ - env/?
 """
 
 
@@ -4307,7 +4084,7 @@ DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
 
 DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
     'app',
-    'tag',  # <rev>-<dt>
+    'tag',
 ])
 
 
@@ -4534,14 +4311,28 @@ class DeployGitRepo:
         check.not_in('.', check.non_empty_str(self.path))
 
 
+@dc.dataclass(frozen=True)
+class DeployGitCheckout:
+    repo: DeployGitRepo
+    rev: DeployRev
+
+    subtrees: ta.Optional[ta.Sequence[str]] = None
+
+    def __post_init__(self) -> None:
+        hash(self)
+        check.non_empty_str(self.rev)
+        if self.subtrees is not None:
+            for st in self.subtrees:
+                check.non_empty_str(st)
+
+
 ##
 
 
 @dc.dataclass(frozen=True)
 class DeploySpec:
     app: DeployApp
-    repo: DeployGitRepo
-    rev: DeployRev
+    checkout: DeployGitCheckout
 
     def __post_init__(self) -> None:
         hash(self)
@@ -6008,9 +5799,7 @@ inj = Injection
 """
 TODO:
  - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
- - namedtuple
  - literals
- - newtypes?
 """
 
 
@@ -6020,7 +5809,7 @@ TODO:
 @dc.dataclass(frozen=True)
 class ObjMarshalOptions:
     raw_bytes: bool = False
-    nonstrict_dataclasses: bool = False
+    non_strict_fields: bool = False
 
 
 class ObjMarshaler(abc.ABC):
@@ -6149,10 +5938,10 @@ class IterableObjMarshaler(ObjMarshaler):
 
 
 @dc.dataclass(frozen=True)
-class DataclassObjMarshaler(ObjMarshaler):
+class FieldsObjMarshaler(ObjMarshaler):
     ty: type
     fs: ta.Mapping[str, ObjMarshaler]
-    nonstrict: bool = False
+    non_strict: bool = False
 
     def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
         return {
@@ -6164,7 +5953,7 @@ class DataclassObjMarshaler(ObjMarshaler):
         return self.ty(**{
             k: self.fs[k].unmarshal(v, ctx)
             for k, v in o.items()
-            if not (self.nonstrict or ctx.options.nonstrict_dataclasses) or k in self.fs
+            if not (self.non_strict or ctx.options.non_strict_fields) or k in self.fs
         })
 
 
@@ -6296,7 +6085,7 @@ class ObjMarshalerManager:
             ty: ta.Any,
             rec: ta.Callable[[ta.Any], ObjMarshaler],
             *,
-            nonstrict_dataclasses: bool = False,
+            non_strict_fields: bool = False,
     ) -> ObjMarshaler:
         if isinstance(ty, type):
             if abc.ABC in ty.__bases__:
@@ -6318,12 +6107,22 @@ class ObjMarshalerManager:
                 return EnumObjMarshaler(ty)
 
             if dc.is_dataclass(ty):
-                return DataclassObjMarshaler(
+                return FieldsObjMarshaler(
                     ty,
                     {f.name: rec(f.type) for f in dc.fields(ty)},
-                    nonstrict=nonstrict_dataclasses,
+                    non_strict=non_strict_fields,
                 )
 
+            if issubclass(ty, tuple) and hasattr(ty, '_fields'):
+                return FieldsObjMarshaler(
+                    ty,
+                    {p.name: rec(p.annotation) for p in inspect.signature(ty).parameters.values()},
+                    non_strict=non_strict_fields,
+                )
+
+        if is_new_type(ty):
+            return rec(get_new_type_supertype(ty))
+
         if is_generic_alias(ty):
             try:
                 mt = self._generic_mapping_types[ta.get_origin(ty)]
@@ -6519,6 +6318,201 @@ class JsonLogFormatter(logging.Formatter):
         return self._json_dumps(dct)
 
 
+########################################
+# ../../../omlish/os/atomics.py
+
+
+##
+
+
+class AtomicPathSwap(abc.ABC):
+    def __init__(
+            self,
+            kind: AtomicPathSwapKind,
+            dst_path: str,
+            *,
+            auto_commit: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self._kind = kind
+        self._dst_path = dst_path
+        self._auto_commit = auto_commit
+
+        self._state: AtomicPathSwapState = 'open'
+
+    def __repr__(self) -> str:
+        return attr_repr(self, 'kind', 'dst_path', 'tmp_path')
+
+    @property
+    def kind(self) -> AtomicPathSwapKind:
+        return self._kind
+
+    @property
+    def dst_path(self) -> str:
+        return self._dst_path
+
+    @property
+    @abc.abstractmethod
+    def tmp_path(self) -> str:
+        raise NotImplementedError
+
+    #
+
+    @property
+    def state(self) -> AtomicPathSwapState:
+        return self._state
+
+    def _check_state(self, *states: AtomicPathSwapState) -> None:
+        if self._state not in states:
+            raise RuntimeError(f'Atomic path swap not in correct state: {self._state}, {states}')
+
+    #
+
+    @abc.abstractmethod
+    def _commit(self) -> None:
+        raise NotImplementedError
+
+    def commit(self) -> None:
+        if self._state == 'committed':
+            return
+        self._check_state('open')
+        try:
+            self._commit()
+        except Exception:  # noqa
+            self._abort()
+            raise
+        else:
+            self._state = 'committed'
+
+    #
+
+    @abc.abstractmethod
+    def _abort(self) -> None:
+        raise NotImplementedError
+
+    def abort(self) -> None:
+        if self._state == 'aborted':
+            return
+        self._abort()
+        self._state = 'aborted'
+
+    #
+
+    def __enter__(self) -> 'AtomicPathSwap':
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if (
+                exc_type is None and
+                self._auto_commit and
+                self._state == 'open'
+        ):
+            self.commit()
+        else:
+            self.abort()
+
+
+class AtomicPathSwapping(abc.ABC):
+    @abc.abstractmethod
+    def begin_atomic_path_swap(
+            self,
+            kind: AtomicPathSwapKind,
+            dst_path: str,
+            *,
+            name_hint: ta.Optional[str] = None,
+            make_dirs: bool = False,
+            **kwargs: ta.Any,
+    ) -> AtomicPathSwap:
+        raise NotImplementedError
+
+
+##
+
+
+class OsRenameAtomicPathSwap(AtomicPathSwap):
+    def __init__(
+            self,
+            kind: AtomicPathSwapKind,
+            dst_path: str,
+            tmp_path: str,
+            **kwargs: ta.Any,
+    ) -> None:
+        if kind == 'dir':
+            check.state(os.path.isdir(tmp_path))
+        elif kind == 'file':
+            check.state(os.path.isfile(tmp_path))
+        else:
+            raise TypeError(kind)
+
+        super().__init__(
+            kind,
+            dst_path,
+            **kwargs,
+        )
+
+        self._tmp_path = tmp_path
+
+    @property
+    def tmp_path(self) -> str:
+        return self._tmp_path
+
+    def _commit(self) -> None:
+        os.rename(self._tmp_path, self._dst_path)
+
+    def _abort(self) -> None:
+        shutil.rmtree(self._tmp_path, ignore_errors=True)
+
+
+class TempDirAtomicPathSwapping(AtomicPathSwapping):
+    def __init__(
+            self,
+            *,
+            temp_dir: ta.Optional[str] = None,
+            root_dir: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+
+        if root_dir is not None:
+            root_dir = os.path.abspath(root_dir)
+        self._root_dir = root_dir
+        self._temp_dir = temp_dir
+
+    def begin_atomic_path_swap(
+            self,
+            kind: AtomicPathSwapKind,
+            dst_path: str,
+            *,
+            name_hint: ta.Optional[str] = None,
+            make_dirs: bool = False,
+            **kwargs: ta.Any,
+    ) -> AtomicPathSwap:
+        dst_path = os.path.abspath(dst_path)
+        if self._root_dir is not None and not dst_path.startswith(check.non_empty_str(self._root_dir)):
+            raise RuntimeError(f'Atomic path swap dst must be in root dir: {dst_path}, {self._root_dir}')
+
+        dst_dir = os.path.dirname(dst_path)
+        if make_dirs:
+            os.makedirs(dst_dir, exist_ok=True)
+        if not os.path.isdir(dst_dir):
+            raise RuntimeError(f'Atomic path swap dst dir does not exist: {dst_dir}')
+
+        if kind == 'dir':
+            tmp_path = tempfile.mkdtemp(prefix=name_hint, dir=self._temp_dir)
+        elif kind == 'file':
+            fd, tmp_path = tempfile.mkstemp(prefix=name_hint, dir=self._temp_dir)
+            os.close(fd)
+        else:
+            raise TypeError(kind)
+
+        return OsRenameAtomicPathSwap(
+            kind,
+            dst_path,
+            tmp_path,
+            **kwargs,
+        )
+
+
 ########################################
 # ../../../omdev/interp/types.py
 
@@ -6749,34 +6743,13 @@ class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
 CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
 
 
-########################################
-# ../deploy/commands.py
-
-
-##
-
-
-@dc.dataclass(frozen=True)
-class DeployCommand(Command['DeployCommand.Output']):
-    @dc.dataclass(frozen=True)
-    class Output(Command.Output):
-        pass
-
-
-class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
-    async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
-        log.info('Deploying!')
-
-        return DeployCommand.Output()
-
-
 ########################################
 # ../deploy/tmp.py
 
 
 class DeployTmpManager(
     SingleDirDeployPathOwner,
-    DeployAtomicPathSwapping,
+    AtomicPathSwapping,
 ):
     def __init__(
             self,
@@ -6789,18 +6762,18 @@ class DeployTmpManager(
         )
 
     @cached_nullary
-    def _swapping(self) -> DeployAtomicPathSwapping:
-        return TempDirDeployAtomicPathSwapping(
+    def _swapping(self) -> AtomicPathSwapping:
+        return TempDirAtomicPathSwapping(
             temp_dir=self._make_dir(),
             root_dir=check.non_empty_str(self._deploy_home),
         )
 
     def begin_atomic_path_swap(
             self,
-            kind: DeployAtomicPathSwapKind,
+            kind: AtomicPathSwapKind,
             dst_path: str,
             **kwargs: ta.Any,
-    ) -> DeployAtomicPathSwap:
+    ) -> AtomicPathSwap:
         return self._swapping().begin_atomic_path_swap(
             kind,
             dst_path,
@@ -8199,7 +8172,7 @@ class DeployGitManager(SingleDirDeployPathOwner):
             self,
             *,
             deploy_home: ta.Optional[DeployHome] = None,
-            atomics: DeployAtomicPathSwapping,
+            atomics: AtomicPathSwapping,
     ) -> None:
         super().__init__(
             owned_dir='git',
@@ -8237,12 +8210,16 @@ class DeployGitManager(SingleDirDeployPathOwner):
             else:
                 return f'https://{self._repo.host}/{self._repo.path}'
 
+        #
+
         async def _call(self, *cmd: str) -> None:
             await asyncio_subprocesses.check_call(
                 *cmd,
                 cwd=self._dir,
             )
 
+        #
+
         @async_cached_nullary
         async def init(self) -> None:
             os.makedirs(self._dir, exist_ok=True)
@@ -8256,7 +8233,9 @@ class DeployGitManager(SingleDirDeployPathOwner):
             await self.init()
             await self._call('git', 'fetch', '--depth=1', 'origin', rev)
 
-        async def checkout(self, rev: DeployRev, dst_dir: str) -> None:
+        #
+
+        async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
             check.state(not os.path.exists(dst_dir))
             with self._git._atomics.begin_atomic_path_swap(  # noqa
                     'dir',
@@ -8264,14 +8243,14 @@ class DeployGitManager(SingleDirDeployPathOwner):
                     auto_commit=True,
                     make_dirs=True,
             ) as dst_swap:
-                await self.fetch(rev)
+                await self.fetch(checkout.rev)
 
                 dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
                 await dst_call('git', 'init')
 
                 await dst_call('git', 'remote', 'add', 'local', self._dir)
-                await dst_call('git', 'fetch', '--depth=1', 'local', rev)
-                await dst_call('git', 'checkout', rev)
+                await dst_call('git', 'fetch', '--depth=1', 'local', checkout.rev)
+                await dst_call('git', 'checkout', checkout.rev, *(checkout.subtrees or []))
 
     def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
         try:
@@ -8280,8 +8259,8 @@ class DeployGitManager(SingleDirDeployPathOwner):
             repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
             return repo_dir
 
-    async def checkout(self, repo: DeployGitRepo, rev: DeployRev, dst_dir: str) -> None:
-        await self.get_repo_dir(repo).checkout(rev, dst_dir)
+    async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
+        await self.get_repo_dir(checkout.repo).checkout(checkout, dst_dir)
 
 
 ########################################
@@ -8298,7 +8277,7 @@ class DeployVenvManager(DeployPathOwner):
             self,
             *,
             deploy_home: ta.Optional[DeployHome] = None,
-            atomics: DeployAtomicPathSwapping,
+            atomics: AtomicPathSwapping,
     ) -> None:
         super().__init__()
 
@@ -8919,15 +8898,14 @@ class DeployAppManager(DeployPathOwner):
     async def prepare_app(
             self,
             spec: DeploySpec,
-    ):
-        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev, spec.key()))
+    ) -> None:
+        app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.checkout.rev, spec.key()))
         app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
 
         #
 
         await self._git.checkout(
-            spec.repo,
-            spec.rev,
+            spec.checkout,
             app_dir,
         )
 
@@ -9650,6 +9628,34 @@ class SystemInterpProvider(InterpProvider):
         raise KeyError(version)
 
 
+########################################
+# ../deploy/commands.py
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployCommand(Command['DeployCommand.Output']):
+    spec: DeploySpec
+
+    @dc.dataclass(frozen=True)
+    class Output(Command.Output):
+        pass
+
+
+@dc.dataclass(frozen=True)
+class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
+    _apps: DeployAppManager
+
+    async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
+        log.info('Deploying! %r', cmd.spec)
+
+        await self._apps.prepare_app(cmd.spec)
+
+        return DeployCommand.Output()
+
+
 ########################################
 # ../remote/inject.py
 
@@ -9815,7 +9821,7 @@ class DockerManageTargetConnector(ManageTargetConnector):
         if dmt.image is not None:
             sh_parts.extend(['run', '-i', dmt.image])
         elif dmt.container_id is not None:
-            sh_parts.extend(['exec', dmt.container_id])
+            sh_parts.extend(['exec', '-i', dmt.container_id])
         else:
             raise ValueError(dmt)
 
@@ -10035,7 +10041,7 @@ def bind_deploy(
         inj.bind(DeployGitManager, singleton=True),
 
         inj.bind(DeployTmpManager, singleton=True),
-        inj.bind(DeployAtomicPathSwapping, to_key=DeployTmpManager),
+        inj.bind(AtomicPathSwapping, to_key=DeployTmpManager),
 
         inj.bind(DeployVenvManager, singleton=True),