ominfra 0.0.0.dev131__py3-none-any.whl → 0.0.0.dev133__py3-none-any.whl

ominfra/{deploy → manage/deploy}/_executor.py

@@ -82,10 +82,10 @@ if sys.version_info < (3, 8):
 ########################################
 
 
-# ../../../omlish/lite/cached.py
+# ../../../../omlish/lite/cached.py
 T = ta.TypeVar('T')
 
-# ../../../omlish/lite/check.py
+# ../../../../omlish/lite/check.py
 SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
 
 
@@ -112,7 +112,7 @@ class HostConfig:
 
 
 ########################################
-# ../../../../omlish/lite/cached.py
+# ../../../../../omlish/lite/cached.py
 
 
 class _cached_nullary:  # noqa
@@ -137,7 +137,7 @@ def cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
 
 
 ########################################
-# ../../../../omlish/lite/check.py
+# ../../../../../omlish/lite/check.py
 
 
 def check_isinstance(v: ta.Any, spec: ta.Union[ta.Type[T], tuple]) -> T:
@@ -234,7 +234,7 @@ def check_non_empty(v: SizedT) -> SizedT:
 
 
 ########################################
-# ../../../../omlish/lite/json.py
+# ../../../../../omlish/lite/json.py
 
 
 ##
@@ -265,7 +265,7 @@ json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON
 
 
 ########################################
-# ../../../../omlish/lite/reflect.py
+# ../../../../../omlish/lite/reflect.py
 
 
 _GENERIC_ALIAS_TYPES = (
@@ -320,7 +320,7 @@ def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
 
 
 ########################################
-# ../../../../omlish/lite/logs.py
+# ../../../../../omlish/lite/logs.py
 """
 TODO:
  - translate json keys
@@ -590,7 +590,7 @@ def configure_standard_logging(
 
 
 ########################################
-# ../../../../omlish/lite/marshal.py
+# ../../../../../omlish/lite/marshal.py
 """
 TODO:
  - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
@@ -934,7 +934,7 @@ def unmarshal_obj(o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
 
 
 ########################################
-# ../../../../omlish/lite/runtime.py
+# ../../../../../omlish/lite/runtime.py
 
 
 @cached_nullary
@@ -951,7 +951,7 @@ def check_runtime_version() -> None:
 
 
 ########################################
-# ../../../../omlish/lite/subprocesses.py
+# ../../../../../omlish/lite/subprocesses.py
 
 
 ##

ominfra/{deploy → manage/deploy}/poly/_main.py

@@ -33,7 +33,7 @@ if sys.version_info < (3, 8):
 ########################################
 
 
-# ../../../omlish/lite/cached.py
+# ../../../../omlish/lite/cached.py
 T = ta.TypeVar('T')
 
 # base.py
@@ -84,7 +84,7 @@ class DeployConfig:
 
 
 ########################################
-# ../../../../omlish/lite/cached.py
+# ../../../../../omlish/lite/cached.py
 
 
 class _cached_nullary:  # noqa
@@ -109,7 +109,7 @@ def cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
 
 
 ########################################
-# ../../../../omlish/lite/json.py
+# ../../../../../omlish/lite/json.py
 
 
 ##
@@ -301,7 +301,7 @@ class Deploy(ConcernsContainer[DeployConcern, DeployConfig]):
 
 
 ########################################
-# ../../../../omlish/lite/logs.py
+# ../../../../../omlish/lite/logs.py
 """
 TODO:
  - translate json keys
@@ -571,7 +571,7 @@ def configure_standard_logging(
 
 
 ########################################
-# ../../../../omlish/lite/runtime.py
+# ../../../../../omlish/lite/runtime.py
 
 
 @cached_nullary
@@ -708,7 +708,7 @@ class SiteImpl(Site):
 
 
 ########################################
-# ../../../../omlish/lite/subprocesses.py
+# ../../../../../omlish/lite/subprocesses.py
 
 
 ##

ominfra/{deploy → manage/deploy}/remote.py

@@ -23,7 +23,7 @@ import tempfile
 
 from omlish import check
 
-from .. import cmds
+from ... import cmds
 
 
 def render_script(*cs: list[str] | tuple[str, ...]) -> str:

ominfra/pyremote.py

@@ -0,0 +1,357 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+Basically this: https://mitogen.networkgenomics.com/howitworks.html
+"""
+import base64
+import dataclasses as dc
+import inspect
+import json
+import os
+import platform
+import pwd
+import site
+import struct
+import sys
+import textwrap
+import typing as ta
+import zlib
+
+from omlish.lite.check import check_isinstance
+from omlish.lite.check import check_none
+
+
+##
+
+
+_PYREMOTE_BOOTSTRAP_COMM_FD = 100
+_PYREMOTE_BOOTSTRAP_SRC_FD = 101
+
+_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CPID'
+_PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
+
+_PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
+_PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
+_PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR001\n'
+_PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR001\n'
+
+_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
+
+_PYREMOTE_BOOTSTRAP_IMPORTS = [
+    'base64',
+    'os',
+    'struct',
+    'sys',
+    'zlib',
+]
+
+
+def _pyremote_bootstrap_main(context_name: str) -> None:
+    # Get pid
+    pid = os.getpid()
+
+    # Two copies of main src to be sent to parent
+    r0, w0 = os.pipe()
+    r1, w1 = os.pipe()
+
+    if (cp := os.fork()):
+        # Parent process
+
+        # Dup original stdin to comm_fd for use as comm channel
+        os.dup2(0, _PYREMOTE_BOOTSTRAP_COMM_FD)
+
+        # Overwrite stdin (fed to python repl) with first copy of src
+        os.dup2(r0, 0)
+
+        # Dup second copy of src to src_fd to recover after launch
+        os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
+
+        # Close remaining fd's
+        for f in [r0, w0, r1, w1]:
+            os.close(f)
+
+        # Save child pid to close after relaunch
+        os.environ[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
+
+        # Save original argv0
+        os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = sys.executable
+
+        # Start repl reading stdin from r0
+        os.execl(sys.executable, sys.executable + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
+
+    else:
+        # Child process
+
+        # Write first ack
+        os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
+
+        # Write pid
+        os.write(1, struct.pack('<Q', pid))
+
+        # Read main src from stdin
+        main_z_len = struct.unpack('<I', os.read(0, 4))[0]
+        main_src = zlib.decompress(os.fdopen(0, 'rb').read(main_z_len))
+
+        # Write both copies of main src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely fill
+        # and block and need to be drained by pyremote_bootstrap_finalize running in parent.
+        for w in [w0, w1]:
+            fp = os.fdopen(w, 'wb', 0)
+            fp.write(main_src)
+            fp.close()
+
+        # Write second ack
+        os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
+
+        # Exit child
+        sys.exit(0)
+
+
+##
+
+
+def pyremote_build_bootstrap_cmd(context_name: str) -> str:
+    bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
+
+    for gl in [
+        '_PYREMOTE_BOOTSTRAP_COMM_FD',
+        '_PYREMOTE_BOOTSTRAP_SRC_FD',
+
+        '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
+        '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
+
+        '_PYREMOTE_BOOTSTRAP_ACK0',
+        '_PYREMOTE_BOOTSTRAP_ACK1',
+
+        '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
+    ]:
+        bs_src = bs_src.replace(gl, repr(globals()[gl]))
+
+    bs_src = '\n'.join(
+        cl
+        for l in bs_src.splitlines()
+        if (cl := (l.split('#')[0]).rstrip())
+        if cl.strip()
+    )
+
+    bs_z = zlib.compress(bs_src.encode('utf-8'))
+    bs_z64 = base64.encodebytes(bs_z).replace(b'\n', b'')
+
+    stmts = [
+        f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
+        f'exec(zlib.decompress(base64.decodebytes({bs_z64!r})))',
+        f'_pyremote_bootstrap_main({context_name!r})',
+    ]
+
+    cmd = '; '.join(stmts)
+    return cmd
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class PyremoteEnvInfo:
+    sys_base_prefix: str
+    sys_byteorder: str
+    sys_defaultencoding: str
+    sys_exec_prefix: str
+    sys_executable: str
+    sys_implementation_name: str
+    sys_path: ta.List[str]
+    sys_platform: str
+    sys_prefix: str
+    sys_version: str
+    sys_version_info: ta.List[ta.Union[int, str]]
+
+    platform_architecture: ta.List[str]
+    platform_machine: str
+    platform_platform: str
+    platform_processor: str
+    platform_system: str
+    platform_release: str
+    platform_version: str
+
+    site_userbase: str
+
+    os_cwd: str
+    os_gid: int
+    os_loadavg: ta.List[float]
+    os_login: ta.Optional[str]
+    os_pgrp: int
+    os_pid: int
+    os_ppid: int
+    os_uid: int
+
+    pw_name: str
+    pw_uid: int
+    pw_gid: int
+    pw_gecos: str
+    pw_dir: str
+    pw_shell: str
+
+    env_path: ta.Optional[str]
+
+
+def _get_pyremote_env_info() -> PyremoteEnvInfo:
+    os_uid = os.getuid()
+
+    pw = pwd.getpwuid(os_uid)
+
+    os_login: ta.Optional[str]
+    try:
+        os_login = os.getlogin()
+    except OSError:
+        os_login = None
+
+    return PyremoteEnvInfo(
+        sys_base_prefix=sys.base_prefix,
+        sys_byteorder=sys.byteorder,
+        sys_defaultencoding=sys.getdefaultencoding(),
+        sys_exec_prefix=sys.exec_prefix,
+        sys_executable=sys.executable,
+        sys_implementation_name=sys.implementation.name,
+        sys_path=sys.path,
+        sys_platform=sys.platform,
+        sys_prefix=sys.prefix,
+        sys_version=sys.version,
+        sys_version_info=list(sys.version_info),
+
+        platform_architecture=list(platform.architecture()),
+        platform_machine=platform.machine(),
+        platform_platform=platform.platform(),
+        platform_processor=platform.processor(),
+        platform_system=platform.system(),
+        platform_release=platform.release(),
+        platform_version=platform.version(),
+
+        site_userbase=site.getuserbase(),
+
+        os_cwd=os.getcwd(),
+        os_gid=os.getgid(),
+        os_loadavg=list(os.getloadavg()),
+        os_login=os_login,
+        os_pgrp=os.getpgrp(),
+        os_pid=os.getpid(),
+        os_ppid=os.getppid(),
+        os_uid=os_uid,
+
+        pw_name=pw.pw_name,
+        pw_uid=pw.pw_uid,
+        pw_gid=pw.pw_gid,
+        pw_gecos=pw.pw_gecos,
+        pw_dir=pw.pw_dir,
+        pw_shell=pw.pw_shell,
+
+        env_path=os.environ.get('PATH'),
+    )
+
+
+##
+
+
+class PyremoteBootstrapDriver:
+    def __init__(self, main_src: str) -> None:
+        super().__init__()
+
+        self._main_src = main_src
+        self._main_z = zlib.compress(main_src.encode('utf-8'))
+
+    @dc.dataclass(frozen=True)
+    class Read:
+        sz: int
+
+    @dc.dataclass(frozen=True)
+    class Write:
+        d: bytes
+
+    class ProtocolError(Exception):
+        pass
+
+    @dc.dataclass(frozen=True)
+    class Result:
+        pid: int
+        env_info: PyremoteEnvInfo
+
+    def __call__(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
+        # Read first ack
+        yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
+
+        # Read pid
+        d = yield from self._read(8)
+        pid = struct.unpack('<Q', d)[0]
+
+        # Write main src
+        check_none((yield self.Write(struct.pack('<I', len(self._main_z)))))
+        check_none((yield self.Write(self._main_z)))
+
+        # Read second and third ack
+        yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
+        yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
+
+        # Read env info
+        d = yield from self._read(4)
+        env_info_json_len = struct.unpack('<I', d)[0]
+        d = yield from self._read(env_info_json_len)
+        env_info_json = d.decode('utf-8')
+        env_info = PyremoteEnvInfo(**json.loads(env_info_json))
+
+        # Read fourth ack
+        yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
+
+        # Return
+        return self.Result(
+            pid=pid,
+            env_info=env_info,
+        )
+
+    def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
+        d = check_isinstance((yield self.Read(sz)), bytes)
+        if len(d) != sz:
+            raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
+        return d
+
+    def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
+        d = yield from self._read(len(e))
+        if d != e:
+            raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class PyremoteBootstrapPayloadRuntime:
+    input: ta.BinaryIO
+    main_src: str
+
+
+def pyremote_bootstrap_finalize() -> PyremoteBootstrapPayloadRuntime:
+    # Restore original argv0
+    sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
+
+    # Read second copy of main src
+    r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
+    main_src = r1.read().decode('utf-8')
+    r1.close()
+
+    # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in a
+    # pipe at once.
+    os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
+
+    # Write third ack
+    os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
+
+    # Write env info
+    env_info = _get_pyremote_env_info()
+    env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':'))  # noqa
+    os.write(1, struct.pack('<I', len(env_info_json)))
+    os.write(1, env_info_json.encode('utf-8'))
+
+    # Write fourth ack
+    os.write(1, _PYREMOTE_BOOTSTRAP_ACK3)
+
+    # Return
+    return PyremoteBootstrapPayloadRuntime(
+        input=os.fdopen(_PYREMOTE_BOOTSTRAP_COMM_FD, 'rb', 0),
+        main_src=main_src,
+    )

ominfra/scripts/journald2aws.py

@@ -1627,6 +1627,14 @@ class ExitStacked:
 ##
 
 
+@contextlib.contextmanager
+def defer(fn: ta.Callable) -> ta.Generator[ta.Callable, None, None]:
+    try:
+        yield fn
+    finally:
+        fn()
+
+
 @contextlib.contextmanager
 def attr_setting(obj, attr, val, *, default=None):  # noqa
     not_set = object()