ominfra 0.0.0.dev145__tar.gz → 0.0.0.dev146__tar.gz

{ominfra-0.0.0.dev145/ominfra.egg-info → ominfra-0.0.0.dev146}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev145
+Version: 0.0.0.dev146
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev145
-Requires-Dist: omlish==0.0.0.dev145
+Requires-Dist: omdev==0.0.0.dev146
+Requires-Dist: omlish==0.0.0.dev146
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.18; extra == "all"

{ominfra-0.0.0.dev145 → ominfra-0.0.0.dev146}/ominfra/manage/commands/inject.py

@@ -20,6 +20,8 @@ from .base import CommandRegistrations
 from .base import build_command_name_map
 from .execution import CommandExecutorMap
 from .execution import LocalCommandExecutor
+from .interp import InterpCommand
+from .interp import InterpCommandExecutor
 from .marshal import install_command_marshaling
 from .subprocess import SubprocessCommand
 from .subprocess import SubprocessCommandExecutor
@@ -111,8 +113,11 @@ def bind_commands(
 
     #
 
+    command_cls: ta.Any
+    executor_cls: ta.Any
     for command_cls, executor_cls in [
         (SubprocessCommand, SubprocessCommandExecutor),
+        (InterpCommand, InterpCommandExecutor),
     ]:
         lst.append(bind_command(command_cls, executor_cls))
 

ominfra-0.0.0.dev146/ominfra/manage/commands/interp.py

@@ -0,0 +1,39 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+
+from omdev.interp.resolvers import DEFAULT_INTERP_RESOLVER
+from omdev.interp.types import InterpOpts
+from omdev.interp.types import InterpSpecifier
+from omlish.lite.check import check_not_none
+
+from ..commands.base import Command
+from ..commands.base import CommandExecutor
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class InterpCommand(Command['InterpCommand.Output']):
+    spec: str
+    install: bool = False
+
+    @dc.dataclass(frozen=True)
+    class Output(Command.Output):
+        exe: str
+        version: str
+        opts: InterpOpts
+
+
+##
+
+
+class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
+    def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
+        i = InterpSpecifier.parse(check_not_none(cmd.spec))
+        o = check_not_none(DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
+        return InterpCommand.Output(
+            exe=o.exe,
+            version=str(o.version.version),
+            opts=o.version.opts,
+        )

{ominfra-0.0.0.dev145 → ominfra-0.0.0.dev146}/ominfra/manage/commands/subprocess.py

@@ -5,6 +5,7 @@ import subprocess
 import time
 import typing as ta
 
+from omlish.lite.check import check_not_isinstance
 from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
 from omlish.lite.subprocesses import SubprocessChannelOption
 from omlish.lite.subprocesses import subprocess_maybe_shell_wrap_exec
@@ -31,8 +32,7 @@ class SubprocessCommand(Command['SubprocessCommand.Output']):
     timeout: ta.Optional[float] = None
 
     def __post_init__(self) -> None:
-        if isinstance(self.cmd, str):
-            raise TypeError(self.cmd)
+        check_not_isinstance(self.cmd, str)
 
     @dc.dataclass(frozen=True)
     class Output(Command.Output):

{ominfra-0.0.0.dev145 → ominfra-0.0.0.dev146}/ominfra/manage/deploy/command.py

@@ -1,6 +1,8 @@
 # ruff: noqa: UP006 UP007
 import dataclasses as dc
 
+from omlish.lite.logs import log
+
 from ..commands.base import Command
 from ..commands.base import CommandExecutor
 
@@ -20,4 +22,6 @@ class DeployCommand(Command['DeployCommand.Output']):
 
 class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
     def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
+        log.info('Deploying!')
+
         return DeployCommand.Output()

ominfra-0.0.0.dev146/ominfra/manage/deploy/paths.py

@@ -0,0 +1,180 @@
+# ruff: noqa: UP006 UP007
+"""
+~deploy
+  deploy.pid (flock)
+  /app
+    /<appspec> - shallow clone
+  /conf
+    /env
+      <appspec>.env
+    /nginx
+      <appspec>.conf
+    /supervisor
+      <appspec>.conf
+  /venv
+    /<appspec>
+
+?
+  /logs
+    /wrmsr--omlish--<spec>
+
+spec = <name>--<rev>--<when>
+
+==
+
+for dn in [
+    'app',
+    'conf',
+    'conf/env',
+    'conf/nginx',
+    'conf/supervisor',
+    'venv',
+]:
+"""
+import abc
+import dataclasses as dc
+import os.path
+import typing as ta
+
+from omlish.lite.check import check_equal
+from omlish.lite.check import check_non_empty
+from omlish.lite.check import check_non_empty_str
+from omlish.lite.check import check_not_in
+
+
+DeployPathKind = ta.Literal['dir', 'file']  # ta.TypeAlias
+
+
+##
+
+
+DEPLOY_PATH_SPEC_PLACEHOLDER = '@'
+
+
+@dc.dataclass(frozen=True)
+class DeployPathPart(abc.ABC):  # noqa
+    @property
+    @abc.abstractmethod
+    def kind(self) -> DeployPathKind:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def render(self) -> str:
+        raise NotImplementedError
+
+
+#
+
+
+class DeployPathDir(DeployPathPart, abc.ABC):
+    @property
+    def kind(self) -> DeployPathKind:
+        return 'dir'
+
+    @classmethod
+    def parse(cls, s: str) -> 'DeployPathDir':
+        if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
+            check_equal(s, DEPLOY_PATH_SPEC_PLACEHOLDER)
+            return SpecDeployPathDir()
+        else:
+            return ConstDeployPathDir(s)
+
+
+class DeployPathFile(DeployPathPart, abc.ABC):
+    @property
+    def kind(self) -> DeployPathKind:
+        return 'file'
+
+    @classmethod
+    def parse(cls, s: str) -> 'DeployPathFile':
+        if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
+            check_equal(s[0], DEPLOY_PATH_SPEC_PLACEHOLDER)
+            return SpecDeployPathFile(s[1:])
+        else:
+            return ConstDeployPathFile(s)
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class ConstDeployPathPart(DeployPathPart, abc.ABC):
+    name: str
+
+    def __post_init__(self) -> None:
+        check_non_empty_str(self.name)
+        check_not_in('/', self.name)
+        check_not_in(DEPLOY_PATH_SPEC_PLACEHOLDER, self.name)
+
+    def render(self) -> str:
+        return self.name
+
+
+class ConstDeployPathDir(ConstDeployPathPart, DeployPathDir):
+    pass
+
+
+class ConstDeployPathFile(ConstDeployPathPart, DeployPathFile):
+    pass
+
+
+#
+
+
+class SpecDeployPathPart(DeployPathPart, abc.ABC):
+    pass
+
+
+class SpecDeployPathDir(SpecDeployPathPart, DeployPathDir):
+    def render(self) -> str:
+        return DEPLOY_PATH_SPEC_PLACEHOLDER
+
+
+@dc.dataclass(frozen=True)
+class SpecDeployPathFile(SpecDeployPathPart, DeployPathFile):
+    suffix: str
+
+    def __post_init__(self) -> None:
+        check_non_empty_str(self.suffix)
+        check_not_in('/', self.suffix)
+        check_not_in(DEPLOY_PATH_SPEC_PLACEHOLDER, self.suffix)
+
+    def render(self) -> str:
+        return DEPLOY_PATH_SPEC_PLACEHOLDER + self.suffix
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployPath:
+    parts: ta.Sequence[DeployPathPart]
+
+    def __post_init__(self) -> None:
+        check_non_empty(self.parts)
+        for p in self.parts[:-1]:
+            check_equal(p.kind, 'dir')
+
+    @property
+    def kind(self) -> ta.Literal['file', 'dir']:
+        return self.parts[-1].kind
+
+    def render(self) -> str:
+        return os.path.join(  # noqa
+            *[p.render() for p in self.parts],
+            *([''] if self.kind == 'dir' else []),
+        )
+
+    @classmethod
+    def parse(cls, s: str) -> 'DeployPath':
+        tail_parse: ta.Callable[[str], DeployPathPart]
+        if s.endswith('/'):
+            tail_parse = DeployPathDir.parse
+            s = s[:-1]
+        else:
+            tail_parse = DeployPathFile.parse
+        ps = check_non_empty_str(s).split('/')
+        return cls([
+            *([DeployPathDir.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
+            tail_parse(ps[-1]),
+        ])

{ominfra-0.0.0.dev145 → ominfra-0.0.0.dev146}/ominfra/manage/main.py

@@ -6,6 +6,7 @@ manage.py -s 'docker run -i python:3.12'
 manage.py -s 'ssh -i /foo/bar.pem foo@bar.baz' -q --python=python3.8
 """
 import contextlib
+import json
 import typing as ta
 
 from omlish.lite.logs import log  # noqa
@@ -18,9 +19,7 @@ from .bootstrap_ import main_bootstrap
 from .commands.base import Command
 from .commands.base import CommandExecutor
 from .commands.execution import LocalCommandExecutor
-from .commands.subprocess import SubprocessCommand
 from .config import MainConfig
-from .deploy.command import DeployCommand
 from .remote.config import RemoteConfig
 from .remote.execution import RemoteExecution
 from .remote.spawning import RemoteSpawning
@@ -78,12 +77,15 @@ def _main() -> None:
 
     #
 
+    msh = injector[ObjMarshalerManager]
+
     cmds: ta.List[Command] = []
+    cmd: Command
     for c in args.command:
-        if c == 'deploy':
-            cmds.append(DeployCommand())
-        else:
-            cmds.append(SubprocessCommand([c]))
+        if not c.startswith('{'):
+            c = json.dumps({c: {}})
+        cmd = msh.unmarshal_obj(json.loads(c), Command)
+        cmds.append(cmd)
 
     #
 
@@ -103,9 +105,13 @@ def _main() -> None:
             ce = es.enter_context(injector[RemoteExecution].connect(tgt, bs))  # noqa
 
         for cmd in cmds:
-            r = ce.try_execute(cmd)
+            r = ce.try_execute(
+                cmd,
+                log=log,
+                omit_exc_object=True,
+            )
 
-            print(injector[ObjMarshalerManager].marshal_obj(r, opts=ObjMarshalOptions(raw_bytes=True)))
+            print(msh.marshal_obj(r, opts=ObjMarshalOptions(raw_bytes=True)))
 
 
 if __name__ == '__main__':

{ominfra-0.0.0.dev145 → ominfra-0.0.0.dev146}/ominfra/manage/remote/channel.py

@@ -1,6 +1,7 @@
 # ruff: noqa: UP006 UP007
 import json
 import struct
+import threading
 import typing as ta
 
 from omlish.lite.json import json_dumps_compact
@@ -25,10 +26,12 @@ class RemoteChannel:
         self._output = output
         self._msh = msh
 
+        self._lock = threading.RLock()
+
     def set_marshaler(self, msh: ObjMarshalerManager) -> None:
         self._msh = msh
 
-    def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
+    def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
         j = json_dumps_compact(self._msh.marshal_obj(o, ty))
         d = j.encode('utf-8')
 
@@ -36,7 +39,11 @@ class RemoteChannel:
         self._output.write(d)
         self._output.flush()
 
-    def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
+    def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
+        with self._lock:
+            return self._send_obj(o, ty)
+
+    def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
         d = self._input.read(4)
         if not d:
             return None
@@ -50,3 +57,7 @@ class RemoteChannel:
 
         j = json.loads(d.decode('utf-8'))
         return self._msh.unmarshal_obj(j, ty)
+
+    def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
+        with self._lock:
+            return self._recv_obj(ty)

{ominfra-0.0.0.dev145 → ominfra-0.0.0.dev146}/ominfra/manage/remote/execution.py

@@ -2,6 +2,7 @@
 import contextlib
 import dataclasses as dc
 import logging
+import threading
 import typing as ta
 
 from omlish.lite.cached import cached_nullary
@@ -36,6 +37,32 @@ else:
 ##
 
 
+class _RemoteExecutionLogHandler(logging.Handler):
+    def __init__(self, fn: ta.Callable[[str], None]) -> None:
+        super().__init__()
+        self._fn = fn
+
+    def emit(self, record):
+        msg = self.format(record)
+        self._fn(msg)
+
+
+@dc.dataclass(frozen=True)
+class _RemoteExecutionRequest:
+    c: Command
+
+
+@dc.dataclass(frozen=True)
+class _RemoteExecutionLog:
+    s: str
+
+
+@dc.dataclass(frozen=True)
+class _RemoteExecutionResponse:
+    r: ta.Optional[CommandOutputOrExceptionData] = None
+    l: ta.Optional[_RemoteExecutionLog] = None
+
+
 def _remote_execution_main() -> None:
     rt = pyremote_bootstrap_finalize()  # noqa
 
@@ -53,20 +80,44 @@ def _remote_execution_main() -> None:
 
     chan.set_marshaler(injector[ObjMarshalerManager])
 
+    #
+
+    log_lock = threading.RLock()
+    send_logs = False
+
+    def log_fn(s: str) -> None:
+        with log_lock:
+            if send_logs:
+                chan.send_obj(_RemoteExecutionResponse(l=_RemoteExecutionLog(s)))
+
+    log_handler = _RemoteExecutionLogHandler(log_fn)
+    logging.root.addHandler(log_handler)
+
+    #
+
     ce = injector[LocalCommandExecutor]
 
     while True:
-        i = chan.recv_obj(Command)
-        if i is None:
+        req = chan.recv_obj(_RemoteExecutionRequest)
+        if req is None:
             break
 
+        with log_lock:
+            send_logs = True
+
         r = ce.try_execute(
-            i,
+            req.c,
             log=log,
             omit_exc_object=True,
         )
 
-        chan.send_obj(r)
+        with log_lock:
+            send_logs = False
+
+        chan.send_obj(_RemoteExecutionResponse(r=CommandOutputOrExceptionData(
+            output=r.output,
+            exception=r.exception,
+        )))
 
 
 ##
@@ -84,12 +135,17 @@ class RemoteCommandExecutor(CommandExecutor):
         self._chan = chan
 
     def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
-        self._chan.send_obj(cmd, Command)
+        self._chan.send_obj(_RemoteExecutionRequest(cmd))
+
+        while True:
+            if (r := self._chan.recv_obj(_RemoteExecutionResponse)) is None:
+                raise EOFError
 
-        if (r := self._chan.recv_obj(CommandOutputOrExceptionData)) is None:
-            raise EOFError
+            if r.l is not None:
+                log.info(r.l.s)
 
-        return r
+            if r.r is not None:
+                return r.r
 
     # @ta.override
     def execute(self, cmd: Command) -> Command.Output: