ominfra 0.0.0.dev132__tar.gz → 0.0.0.dev133__tar.gz

{ominfra-0.0.0.dev132/ominfra.egg-info → ominfra-0.0.0.dev133}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev132
+Version: 0.0.0.dev133
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev132
-Requires-Dist: omlish==0.0.0.dev132
+Requires-Dist: omdev==0.0.0.dev133
+Requires-Dist: omlish==0.0.0.dev133
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.18; extra == "all"

{ominfra-0.0.0.dev132/ominfra → ominfra-0.0.0.dev133/ominfra/manage}/deploy/_executor.py

@@ -82,10 +82,10 @@ if sys.version_info < (3, 8):
 ########################################
 
 
-# ../../../omlish/lite/cached.py
+# ../../../../omlish/lite/cached.py
 T = ta.TypeVar('T')
 
-# ../../../omlish/lite/check.py
+# ../../../../omlish/lite/check.py
 SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
 
 
@@ -112,7 +112,7 @@ class HostConfig:
 
 
 ########################################
-# ../../../../omlish/lite/cached.py
+# ../../../../../omlish/lite/cached.py
 
 
 class _cached_nullary:  # noqa
@@ -137,7 +137,7 @@ def cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
 
 
 ########################################
-# ../../../../omlish/lite/check.py
+# ../../../../../omlish/lite/check.py
 
 
 def check_isinstance(v: ta.Any, spec: ta.Union[ta.Type[T], tuple]) -> T:
@@ -234,7 +234,7 @@ def check_non_empty(v: SizedT) -> SizedT:
 
 
 ########################################
-# ../../../../omlish/lite/json.py
+# ../../../../../omlish/lite/json.py
 
 
 ##
@@ -265,7 +265,7 @@ json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON
 
 
 ########################################
-# ../../../../omlish/lite/reflect.py
+# ../../../../../omlish/lite/reflect.py
 
 
 _GENERIC_ALIAS_TYPES = (
@@ -320,7 +320,7 @@ def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
 
 
 ########################################
-# ../../../../omlish/lite/logs.py
+# ../../../../../omlish/lite/logs.py
 """
 TODO:
  - translate json keys
@@ -590,7 +590,7 @@ def configure_standard_logging(
 
 
 ########################################
-# ../../../../omlish/lite/marshal.py
+# ../../../../../omlish/lite/marshal.py
 """
 TODO:
  - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
@@ -934,7 +934,7 @@ def unmarshal_obj(o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
 
 
 ########################################
-# ../../../../omlish/lite/runtime.py
+# ../../../../../omlish/lite/runtime.py
 
 
 @cached_nullary
@@ -951,7 +951,7 @@ def check_runtime_version() -> None:
 
 
 ########################################
-# ../../../../omlish/lite/subprocesses.py
+# ../../../../../omlish/lite/subprocesses.py
 
 
 ##

{ominfra-0.0.0.dev132/ominfra → ominfra-0.0.0.dev133/ominfra/manage}/deploy/poly/_main.py

@@ -33,7 +33,7 @@ if sys.version_info < (3, 8):
 ########################################
 
 
-# ../../../omlish/lite/cached.py
+# ../../../../omlish/lite/cached.py
 T = ta.TypeVar('T')
 
 # base.py
@@ -84,7 +84,7 @@ class DeployConfig:
 
 
 ########################################
-# ../../../../omlish/lite/cached.py
+# ../../../../../omlish/lite/cached.py
 
 
 class _cached_nullary:  # noqa
@@ -109,7 +109,7 @@ def cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
 
 
 ########################################
-# ../../../../omlish/lite/json.py
+# ../../../../../omlish/lite/json.py
 
 
 ##
@@ -301,7 +301,7 @@ class Deploy(ConcernsContainer[DeployConcern, DeployConfig]):
 
 
 ########################################
-# ../../../../omlish/lite/logs.py
+# ../../../../../omlish/lite/logs.py
 """
 TODO:
  - translate json keys
@@ -571,7 +571,7 @@ def configure_standard_logging(
 
 
 ########################################
-# ../../../../omlish/lite/runtime.py
+# ../../../../../omlish/lite/runtime.py
 
 
 @cached_nullary
@@ -708,7 +708,7 @@ class SiteImpl(Site):
 
 
 ########################################
-# ../../../../omlish/lite/subprocesses.py
+# ../../../../../omlish/lite/subprocesses.py
 
 
 ##

{ominfra-0.0.0.dev132/ominfra → ominfra-0.0.0.dev133/ominfra/manage}/deploy/remote.py

@@ -23,7 +23,7 @@ import tempfile
 
 from omlish import check
 
-from .. import cmds
+from ... import cmds
 
 
 def render_script(*cs: list[str] | tuple[str, ...]) -> str:

ominfra-0.0.0.dev133/ominfra/pyremote.py

@@ -0,0 +1,357 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+Basically this: https://mitogen.networkgenomics.com/howitworks.html
+"""
+import base64
+import dataclasses as dc
+import inspect
+import json
+import os
+import platform
+import pwd
+import site
+import struct
+import sys
+import textwrap
+import typing as ta
+import zlib
+
+from omlish.lite.check import check_isinstance
+from omlish.lite.check import check_none
+
+
+##
+
+
+_PYREMOTE_BOOTSTRAP_COMM_FD = 100
+_PYREMOTE_BOOTSTRAP_SRC_FD = 101
+
+_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CPID'
+_PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
+
+_PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
+_PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
+_PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR001\n'
+_PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR001\n'
+
+_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
+
+_PYREMOTE_BOOTSTRAP_IMPORTS = [
+    'base64',
+    'os',
+    'struct',
+    'sys',
+    'zlib',
+]
+
+
+def _pyremote_bootstrap_main(context_name: str) -> None:
+    # Get pid
+    pid = os.getpid()
+
+    # Two copies of main src to be sent to parent
+    r0, w0 = os.pipe()
+    r1, w1 = os.pipe()
+
+    if (cp := os.fork()):
+        # Parent process
+
+        # Dup original stdin to comm_fd for use as comm channel
+        os.dup2(0, _PYREMOTE_BOOTSTRAP_COMM_FD)
+
+        # Overwrite stdin (fed to python repl) with first copy of src
+        os.dup2(r0, 0)
+
+        # Dup second copy of src to src_fd to recover after launch
+        os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
+
+        # Close remaining fd's
+        for f in [r0, w0, r1, w1]:
+            os.close(f)
+
+        # Save child pid to close after relaunch
+        os.environ[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
+
+        # Save original argv0
+        os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = sys.executable
+
+        # Start repl reading stdin from r0
+        os.execl(sys.executable, sys.executable + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
+
+    else:
+        # Child process
+
+        # Write first ack
+        os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
+
+        # Write pid
+        os.write(1, struct.pack('<Q', pid))
+
+        # Read main src from stdin
+        main_z_len = struct.unpack('<I', os.read(0, 4))[0]
+        main_src = zlib.decompress(os.fdopen(0, 'rb').read(main_z_len))
+
+        # Write both copies of main src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely fill
+        # and block and need to be drained by pyremote_bootstrap_finalize running in parent.
+        for w in [w0, w1]:
+            fp = os.fdopen(w, 'wb', 0)
+            fp.write(main_src)
+            fp.close()
+
+        # Write second ack
+        os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
+
+        # Exit child
+        sys.exit(0)
+
+
+##
+
+
+def pyremote_build_bootstrap_cmd(context_name: str) -> str:
+    bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
+
+    for gl in [
+        '_PYREMOTE_BOOTSTRAP_COMM_FD',
+        '_PYREMOTE_BOOTSTRAP_SRC_FD',
+
+        '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
+        '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
+
+        '_PYREMOTE_BOOTSTRAP_ACK0',
+        '_PYREMOTE_BOOTSTRAP_ACK1',
+
+        '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
+    ]:
+        bs_src = bs_src.replace(gl, repr(globals()[gl]))
+
+    bs_src = '\n'.join(
+        cl
+        for l in bs_src.splitlines()
+        if (cl := (l.split('#')[0]).rstrip())
+        if cl.strip()
+    )
+
+    bs_z = zlib.compress(bs_src.encode('utf-8'))
+    bs_z64 = base64.encodebytes(bs_z).replace(b'\n', b'')
+
+    stmts = [
+        f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
+        f'exec(zlib.decompress(base64.decodebytes({bs_z64!r})))',
+        f'_pyremote_bootstrap_main({context_name!r})',
+    ]
+
+    cmd = '; '.join(stmts)
+    return cmd
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class PyremoteEnvInfo:
+    sys_base_prefix: str
+    sys_byteorder: str
+    sys_defaultencoding: str
+    sys_exec_prefix: str
+    sys_executable: str
+    sys_implementation_name: str
+    sys_path: ta.List[str]
+    sys_platform: str
+    sys_prefix: str
+    sys_version: str
+    sys_version_info: ta.List[ta.Union[int, str]]
+
+    platform_architecture: ta.List[str]
+    platform_machine: str
+    platform_platform: str
+    platform_processor: str
+    platform_system: str
+    platform_release: str
+    platform_version: str
+
+    site_userbase: str
+
+    os_cwd: str
+    os_gid: int
+    os_loadavg: ta.List[float]
+    os_login: ta.Optional[str]
+    os_pgrp: int
+    os_pid: int
+    os_ppid: int
+    os_uid: int
+
+    pw_name: str
+    pw_uid: int
+    pw_gid: int
+    pw_gecos: str
+    pw_dir: str
+    pw_shell: str
+
+    env_path: ta.Optional[str]
+
+
+def _get_pyremote_env_info() -> PyremoteEnvInfo:
+    os_uid = os.getuid()
+
+    pw = pwd.getpwuid(os_uid)
+
+    os_login: ta.Optional[str]
+    try:
+        os_login = os.getlogin()
+    except OSError:
+        os_login = None
+
+    return PyremoteEnvInfo(
+        sys_base_prefix=sys.base_prefix,
+        sys_byteorder=sys.byteorder,
+        sys_defaultencoding=sys.getdefaultencoding(),
+        sys_exec_prefix=sys.exec_prefix,
+        sys_executable=sys.executable,
+        sys_implementation_name=sys.implementation.name,
+        sys_path=sys.path,
+        sys_platform=sys.platform,
+        sys_prefix=sys.prefix,
+        sys_version=sys.version,
+        sys_version_info=list(sys.version_info),
+
+        platform_architecture=list(platform.architecture()),
+        platform_machine=platform.machine(),
+        platform_platform=platform.platform(),
+        platform_processor=platform.processor(),
+        platform_system=platform.system(),
+        platform_release=platform.release(),
+        platform_version=platform.version(),
+
+        site_userbase=site.getuserbase(),
+
+        os_cwd=os.getcwd(),
+        os_gid=os.getgid(),
+        os_loadavg=list(os.getloadavg()),
+        os_login=os_login,
+        os_pgrp=os.getpgrp(),
+        os_pid=os.getpid(),
+        os_ppid=os.getppid(),
+        os_uid=os_uid,
+
+        pw_name=pw.pw_name,
+        pw_uid=pw.pw_uid,
+        pw_gid=pw.pw_gid,
+        pw_gecos=pw.pw_gecos,
+        pw_dir=pw.pw_dir,
+        pw_shell=pw.pw_shell,
+
+        env_path=os.environ.get('PATH'),
+    )
+
+
+##
+
+
+class PyremoteBootstrapDriver:
+    def __init__(self, main_src: str) -> None:
+        super().__init__()
+
+        self._main_src = main_src
+        self._main_z = zlib.compress(main_src.encode('utf-8'))
+
+    @dc.dataclass(frozen=True)
+    class Read:
+        sz: int
+
+    @dc.dataclass(frozen=True)
+    class Write:
+        d: bytes
+
+    class ProtocolError(Exception):
+        pass
+
+    @dc.dataclass(frozen=True)
+    class Result:
+        pid: int
+        env_info: PyremoteEnvInfo
+
+    def __call__(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
+        # Read first ack
+        yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
+
+        # Read pid
+        d = yield from self._read(8)
+        pid = struct.unpack('<Q', d)[0]
+
+        # Write main src
+        check_none((yield self.Write(struct.pack('<I', len(self._main_z)))))
+        check_none((yield self.Write(self._main_z)))
+
+        # Read second and third ack
+        yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
+        yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
+
+        # Read env info
+        d = yield from self._read(4)
+        env_info_json_len = struct.unpack('<I', d)[0]
+        d = yield from self._read(env_info_json_len)
+        env_info_json = d.decode('utf-8')
+        env_info = PyremoteEnvInfo(**json.loads(env_info_json))
+
+        # Read fourth ack
+        yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
+
+        # Return
+        return self.Result(
+            pid=pid,
+            env_info=env_info,
+        )
+
+    def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
+        d = check_isinstance((yield self.Read(sz)), bytes)
+        if len(d) != sz:
+            raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
+        return d
+
+    def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
+        d = yield from self._read(len(e))
+        if d != e:
+            raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class PyremoteBootstrapPayloadRuntime:
+    input: ta.BinaryIO
+    main_src: str
+
+
+def pyremote_bootstrap_finalize() -> PyremoteBootstrapPayloadRuntime:
+    # Restore original argv0
+    sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
+
+    # Read second copy of main src
+    r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
+    main_src = r1.read().decode('utf-8')
+    r1.close()
+
+    # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in a
+    # pipe at once.
+    os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
+
+    # Write third ack
+    os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
+
+    # Write env info
+    env_info = _get_pyremote_env_info()
+    env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':'))  # noqa
+    os.write(1, struct.pack('<I', len(env_info_json)))
+    os.write(1, env_info_json.encode('utf-8'))
+
+    # Write fourth ack
+    os.write(1, _PYREMOTE_BOOTSTRAP_ACK3)
+
+    # Return
+    return PyremoteBootstrapPayloadRuntime(
+        input=os.fdopen(_PYREMOTE_BOOTSTRAP_COMM_FD, 'rb', 0),
+        main_src=main_src,
+    )

{ominfra-0.0.0.dev132 → ominfra-0.0.0.dev133}/ominfra/scripts/supervisor.py

@@ -3,7 +3,7 @@
 # @omlish-lite
 # @omlish-script
 # @omlish-amalg-output ../supervisor/main.py
-# ruff: noqa: N802 U006 UP006 UP007 UP012 UP036
+# ruff: noqa: N802 UP006 UP007 UP012 UP036
 # Supervisor is licensed under the following license:
 #
 #  A copyright notice accompanies this license document that identifies the copyright holders.

{ominfra-0.0.0.dev132 → ominfra-0.0.0.dev133}/ominfra/supervisor/http.py

@@ -1,4 +1,4 @@
-# ruff: noqa: U006 UP007
+# ruff: noqa: UP006 UP007
 import contextlib
 import json
 import socket

{ominfra-0.0.0.dev132 → ominfra-0.0.0.dev133/ominfra.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev132
+Version: 0.0.0.dev133
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev132
-Requires-Dist: omlish==0.0.0.dev132
+Requires-Dist: omdev==0.0.0.dev133
+Requires-Dist: omlish==0.0.0.dev133
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.18; extra == "all"

{ominfra-0.0.0.dev132 → ominfra-0.0.0.dev133}/ominfra.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ ominfra/__about__.py
 ominfra/__init__.py
 ominfra/cmds.py
 ominfra/configs.py
+ominfra/pyremote.py
 ominfra/ssh.py
 ominfra/threadworkers.py
 ominfra.egg-info/PKG-INFO
@@ -31,33 +32,6 @@ ominfra/clouds/aws/journald2aws/main.py
 ominfra/clouds/aws/journald2aws/poster.py
 ominfra/clouds/gcp/__init__.py
 ominfra/clouds/gcp/auth.py
-ominfra/deploy/__init__.py
-ominfra/deploy/_executor.py
-ominfra/deploy/configs.py
-ominfra/deploy/remote.py
-ominfra/deploy/executor/__init__.py
-ominfra/deploy/executor/base.py
-ominfra/deploy/executor/main.py
-ominfra/deploy/executor/concerns/__init__.py
-ominfra/deploy/executor/concerns/dirs.py
-ominfra/deploy/executor/concerns/nginx.py
-ominfra/deploy/executor/concerns/repo.py
-ominfra/deploy/executor/concerns/supervisor.py
-ominfra/deploy/executor/concerns/systemd.py
-ominfra/deploy/executor/concerns/user.py
-ominfra/deploy/executor/concerns/venv.py
-ominfra/deploy/poly/__init__.py
-ominfra/deploy/poly/_main.py
-ominfra/deploy/poly/base.py
-ominfra/deploy/poly/configs.py
-ominfra/deploy/poly/deploy.py
-ominfra/deploy/poly/main.py
-ominfra/deploy/poly/nginx.py
-ominfra/deploy/poly/repo.py
-ominfra/deploy/poly/runtime.py
-ominfra/deploy/poly/site.py
-ominfra/deploy/poly/supervisor.py
-ominfra/deploy/poly/venv.py
 ominfra/journald/__init__.py
 ominfra/journald/fields.py
 ominfra/journald/genmessages.py
@@ -65,10 +39,33 @@ ominfra/journald/messages.py
 ominfra/journald/tailer.py
 ominfra/manage/__init__.py
 ominfra/manage/manage.py
-ominfra/pyremote/__init__.py
-ominfra/pyremote/_runcommands.py
-ominfra/pyremote/bootstrap.py
-ominfra/pyremote/runcommands.py
+ominfra/manage/deploy/__init__.py
+ominfra/manage/deploy/_executor.py
+ominfra/manage/deploy/configs.py
+ominfra/manage/deploy/remote.py
+ominfra/manage/deploy/executor/__init__.py
+ominfra/manage/deploy/executor/base.py
+ominfra/manage/deploy/executor/main.py
+ominfra/manage/deploy/executor/concerns/__init__.py
+ominfra/manage/deploy/executor/concerns/dirs.py
+ominfra/manage/deploy/executor/concerns/nginx.py
+ominfra/manage/deploy/executor/concerns/repo.py
+ominfra/manage/deploy/executor/concerns/supervisor.py
+ominfra/manage/deploy/executor/concerns/systemd.py
+ominfra/manage/deploy/executor/concerns/user.py
+ominfra/manage/deploy/executor/concerns/venv.py
+ominfra/manage/deploy/poly/__init__.py
+ominfra/manage/deploy/poly/_main.py
+ominfra/manage/deploy/poly/base.py
+ominfra/manage/deploy/poly/configs.py
+ominfra/manage/deploy/poly/deploy.py
+ominfra/manage/deploy/poly/main.py
+ominfra/manage/deploy/poly/nginx.py
+ominfra/manage/deploy/poly/repo.py
+ominfra/manage/deploy/poly/runtime.py
+ominfra/manage/deploy/poly/site.py
+ominfra/manage/deploy/poly/supervisor.py
+ominfra/manage/deploy/poly/venv.py
 ominfra/scripts/__init__.py
 ominfra/scripts/journald2aws.py
 ominfra/scripts/supervisor.py

{ominfra-0.0.0.dev132 → ominfra-0.0.0.dev133}/ominfra.egg-info/requires.txt

@@ -1,5 +1,5 @@
-omdev==0.0.0.dev132
-omlish==0.0.0.dev132
+omdev==0.0.0.dev133
+omlish==0.0.0.dev133
 
 [all]
 paramiko~=3.5

{ominfra-0.0.0.dev132 → ominfra-0.0.0.dev133}/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 urls = {source = 'https://github.com/wrmsr/omlish'}
 license = {text = 'BSD-3-Clause'}
 requires-python = '>=3.12'
-version = '0.0.0.dev132'
+version = '0.0.0.dev133'
 classifiers = [
     'License :: OSI Approved :: BSD License',
     'Development Status :: 2 - Pre-Alpha',
@@ -22,8 +22,8 @@ classifiers = [
 ]
 description = 'ominfra'
 dependencies = [
-    'omdev == 0.0.0.dev132',
-    'omlish == 0.0.0.dev132',
+    'omdev == 0.0.0.dev133',
+    'omlish == 0.0.0.dev133',
 ]
 
 [project.optional-dependencies]