ominfra 0.0.0.dev140__tar.gz → 0.0.0.dev142__tar.gz
Sign up to get free protection for your applications and to get access to all the features.
- {ominfra-0.0.0.dev140/ominfra.egg-info → ominfra-0.0.0.dev142}/PKG-INFO +3 -3
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/commands/subprocess.py +18 -17
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/main.py +25 -63
- ominfra-0.0.0.dev142/ominfra/manage/protocol.py +45 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/spawning.py +4 -8
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/pyremote.py +13 -11
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/scripts/journald2aws.py +123 -86
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/scripts/manage.py +219 -176
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/scripts/supervisor.py +110 -86
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142/ominfra.egg-info}/PKG-INFO +3 -3
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra.egg-info/SOURCES.txt +1 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra.egg-info/requires.txt +2 -2
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/pyproject.toml +3 -3
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/LICENSE +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/MANIFEST.in +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/README.rst +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/.manifests.json +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/__about__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/__main__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/auth.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/cli.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/dataclasses.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/__main__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/cursor.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/driver.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/main.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/poster.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/logs.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/metadata.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/gcp/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/gcp/auth.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/cmds.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/configs.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/journald/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/journald/fields.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/journald/genmessages.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/journald/messages.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/journald/tailer.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/commands/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/commands/base.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/payload.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/scripts/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/ssh.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/LICENSE.txt +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/__main__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/configs.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/dispatchers.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/dispatchersimpl.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/events.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/exceptions.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/groups.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/groupsimpl.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/http.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/inject.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/io.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/main.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/pipes.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/privileges.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/process.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/processimpl.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/setup.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/setupimpl.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/signals.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/spawning.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/spawningimpl.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/states.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/supervisor.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/types.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/collections.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/diag.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/fds.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/fs.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/os.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/ostypes.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/signals.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/strings.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/users.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/tailscale/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/tailscale/api.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/tailscale/cli.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/threadworkers.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/tools/__init__.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/tools/listresources.py +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra.egg-info/dependency_links.txt +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra.egg-info/entry_points.txt +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra.egg-info/top_level.txt +0 -0
- {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: ominfra
|
3
|
-
Version: 0.0.0.
|
3
|
+
Version: 0.0.0.dev142
|
4
4
|
Summary: ominfra
|
5
5
|
Author: wrmsr
|
6
6
|
License: BSD-3-Clause
|
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
|
|
12
12
|
Classifier: Operating System :: POSIX
|
13
13
|
Requires-Python: >=3.12
|
14
14
|
License-File: LICENSE
|
15
|
-
Requires-Dist: omdev==0.0.0.
|
16
|
-
Requires-Dist: omlish==0.0.0.
|
15
|
+
Requires-Dist: omdev==0.0.0.dev142
|
16
|
+
Requires-Dist: omlish==0.0.0.dev142
|
17
17
|
Provides-Extra: all
|
18
18
|
Requires-Dist: paramiko~=3.5; extra == "all"
|
19
19
|
Requires-Dist: asyncssh~=2.18; extra == "all"
|
@@ -5,6 +5,8 @@ import subprocess
|
|
5
5
|
import time
|
6
6
|
import typing as ta
|
7
7
|
|
8
|
+
from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
|
9
|
+
from omlish.lite.subprocesses import SubprocessChannelOption
|
8
10
|
from omlish.lite.subprocesses import subprocess_maybe_shell_wrap_exec
|
9
11
|
|
10
12
|
from .base import Command
|
@@ -16,21 +18,21 @@ from .base import CommandExecutor
|
|
16
18
|
|
17
19
|
@dc.dataclass(frozen=True)
|
18
20
|
class SubprocessCommand(Command['SubprocessCommand.Output']):
|
19
|
-
|
21
|
+
cmd: ta.Sequence[str]
|
20
22
|
|
21
23
|
shell: bool = False
|
22
24
|
cwd: ta.Optional[str] = None
|
23
25
|
env: ta.Optional[ta.Mapping[str, str]] = None
|
24
26
|
|
25
|
-
|
26
|
-
|
27
|
+
stdout: str = 'pipe' # SubprocessChannelOption
|
28
|
+
stderr: str = 'pipe' # SubprocessChannelOption
|
27
29
|
|
28
30
|
input: ta.Optional[bytes] = None
|
29
31
|
timeout: ta.Optional[float] = None
|
30
32
|
|
31
33
|
def __post_init__(self) -> None:
|
32
|
-
if isinstance(self.
|
33
|
-
raise TypeError(self.
|
34
|
+
if isinstance(self.cmd, str):
|
35
|
+
raise TypeError(self.cmd)
|
34
36
|
|
35
37
|
@dc.dataclass(frozen=True)
|
36
38
|
class Output(Command.Output):
|
@@ -48,24 +50,23 @@ class SubprocessCommand(Command['SubprocessCommand.Output']):
|
|
48
50
|
|
49
51
|
class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
|
50
52
|
def execute(self, inp: SubprocessCommand) -> SubprocessCommand.Output:
|
51
|
-
|
52
|
-
subprocess_maybe_shell_wrap_exec(*inp.
|
53
|
+
with subprocess.Popen(
|
54
|
+
subprocess_maybe_shell_wrap_exec(*inp.cmd),
|
53
55
|
|
54
56
|
shell=inp.shell,
|
55
57
|
cwd=inp.cwd,
|
56
58
|
env={**os.environ, **(inp.env or {})},
|
57
59
|
|
58
60
|
stdin=subprocess.PIPE if inp.input is not None else None,
|
59
|
-
stdout=
|
60
|
-
stderr=
|
61
|
-
)
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
end_time = time.time()
|
61
|
+
stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stdout)],
|
62
|
+
stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stderr)],
|
63
|
+
) as proc:
|
64
|
+
start_time = time.time()
|
65
|
+
stdout, stderr = proc.communicate(
|
66
|
+
input=inp.input,
|
67
|
+
timeout=inp.timeout,
|
68
|
+
)
|
69
|
+
end_time = time.time()
|
69
70
|
|
70
71
|
return SubprocessCommand.Output(
|
71
72
|
rc=proc.returncode,
|
@@ -5,17 +5,9 @@
|
|
5
5
|
manage.py -s 'docker run -i python:3.12'
|
6
6
|
manage.py -s 'ssh -i /foo/bar.pem foo@bar.baz' -q --python=python3.8
|
7
7
|
"""
|
8
|
-
import
|
9
|
-
import
|
10
|
-
import typing as ta
|
11
|
-
|
12
|
-
from omlish.lite.cached import static_init
|
13
|
-
from omlish.lite.json import json_dumps_compact
|
8
|
+
from omlish.lite.marshal import OBJ_MARSHALER_MANAGER
|
9
|
+
from omlish.lite.marshal import ObjMarshalerManager
|
14
10
|
from omlish.lite.marshal import PolymorphicObjMarshaler
|
15
|
-
from omlish.lite.marshal import get_obj_marshaler
|
16
|
-
from omlish.lite.marshal import marshal_obj
|
17
|
-
from omlish.lite.marshal import register_opj_marshaler
|
18
|
-
from omlish.lite.marshal import unmarshal_obj
|
19
11
|
|
20
12
|
from ..pyremote import PyremoteBootstrapDriver
|
21
13
|
from ..pyremote import PyremoteBootstrapOptions
|
@@ -25,6 +17,7 @@ from .commands.base import Command
|
|
25
17
|
from .commands.subprocess import SubprocessCommand
|
26
18
|
from .commands.subprocess import SubprocessCommandExecutor
|
27
19
|
from .payload import get_payload_src
|
20
|
+
from .protocol import Channel
|
28
21
|
from .spawning import PySpawner
|
29
22
|
|
30
23
|
|
@@ -36,51 +29,28 @@ _COMMAND_TYPES = {
|
|
36
29
|
}
|
37
30
|
|
38
31
|
|
39
|
-
|
40
|
-
|
32
|
+
##
|
33
|
+
|
34
|
+
|
35
|
+
def register_command_marshaling(msh: ObjMarshalerManager) -> None:
|
41
36
|
for fn in [
|
42
37
|
lambda c: c,
|
43
38
|
lambda c: c.Output,
|
44
39
|
]:
|
45
|
-
register_opj_marshaler(
|
40
|
+
msh.register_opj_marshaler(
|
46
41
|
fn(Command),
|
47
42
|
PolymorphicObjMarshaler.of([
|
48
43
|
PolymorphicObjMarshaler.Impl(
|
49
44
|
fn(cty),
|
50
45
|
k,
|
51
|
-
get_obj_marshaler(fn(cty)),
|
46
|
+
msh.get_obj_marshaler(fn(cty)),
|
52
47
|
)
|
53
48
|
for k, cty in _COMMAND_TYPES.items()
|
54
49
|
]),
|
55
50
|
)
|
56
51
|
|
57
52
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
def _send_obj(f: ta.IO, o: ta.Any, ty: ta.Any = None) -> None:
|
62
|
-
j = json_dumps_compact(marshal_obj(o, ty))
|
63
|
-
d = j.encode('utf-8')
|
64
|
-
|
65
|
-
f.write(struct.pack('<I', len(d)))
|
66
|
-
f.write(d)
|
67
|
-
f.flush()
|
68
|
-
|
69
|
-
|
70
|
-
def _recv_obj(f: ta.IO, ty: ta.Any) -> ta.Any:
|
71
|
-
d = f.read(4)
|
72
|
-
if not d:
|
73
|
-
return None
|
74
|
-
if len(d) != 4:
|
75
|
-
raise EOFError
|
76
|
-
|
77
|
-
sz = struct.unpack('<I', d)[0]
|
78
|
-
d = f.read(sz)
|
79
|
-
if len(d) != sz:
|
80
|
-
raise EOFError
|
81
|
-
|
82
|
-
j = json.loads(d.decode('utf-8'))
|
83
|
-
return unmarshal_obj(j, ty)
|
53
|
+
register_command_marshaling(OBJ_MARSHALER_MANAGER)
|
84
54
|
|
85
55
|
|
86
56
|
##
|
@@ -88,9 +58,10 @@ def _recv_obj(f: ta.IO, ty: ta.Any) -> ta.Any:
|
|
88
58
|
|
89
59
|
def _remote_main() -> None:
|
90
60
|
rt = pyremote_bootstrap_finalize() # noqa
|
61
|
+
chan = Channel(rt.input, rt.output)
|
91
62
|
|
92
63
|
while True:
|
93
|
-
i =
|
64
|
+
i = chan.recv_obj(Command)
|
94
65
|
if i is None:
|
95
66
|
break
|
96
67
|
|
@@ -99,7 +70,7 @@ def _remote_main() -> None:
|
|
99
70
|
else:
|
100
71
|
raise TypeError(i)
|
101
72
|
|
102
|
-
|
73
|
+
chan.send_obj(o, Command.Output)
|
103
74
|
|
104
75
|
|
105
76
|
##
|
@@ -110,11 +81,13 @@ def _main() -> None:
|
|
110
81
|
|
111
82
|
parser = argparse.ArgumentParser()
|
112
83
|
|
84
|
+
parser.add_argument('--_payload-file')
|
85
|
+
|
113
86
|
parser.add_argument('-s', '--shell')
|
114
87
|
parser.add_argument('-q', '--shell-quote', action='store_true')
|
115
88
|
parser.add_argument('--python', default='python3')
|
89
|
+
|
116
90
|
parser.add_argument('--debug', action='store_true')
|
117
|
-
parser.add_argument('--_payload-file')
|
118
91
|
|
119
92
|
args = parser.parse_args()
|
120
93
|
|
@@ -122,8 +95,6 @@ def _main() -> None:
|
|
122
95
|
|
123
96
|
payload_src = get_payload_src(file=args._payload_file) # noqa
|
124
97
|
|
125
|
-
#
|
126
|
-
|
127
98
|
remote_src = '\n\n'.join([
|
128
99
|
'__name__ = "__remote__"',
|
129
100
|
payload_src,
|
@@ -132,41 +103,32 @@ def _main() -> None:
|
|
132
103
|
|
133
104
|
#
|
134
105
|
|
135
|
-
bs_src = pyremote_build_bootstrap_cmd(__package__ or 'manage')
|
136
|
-
|
137
|
-
#
|
138
|
-
|
139
106
|
spawner = PySpawner(
|
140
|
-
|
107
|
+
pyremote_build_bootstrap_cmd(__package__ or 'manage'),
|
141
108
|
shell=args.shell,
|
142
109
|
shell_quote=args.shell_quote,
|
143
110
|
python=args.python,
|
144
111
|
)
|
112
|
+
|
145
113
|
with spawner.spawn() as proc:
|
146
114
|
res = PyremoteBootstrapDriver( # noqa
|
147
115
|
remote_src,
|
148
116
|
PyremoteBootstrapOptions(
|
149
117
|
debug=args.debug,
|
150
118
|
),
|
151
|
-
).run(proc.
|
152
|
-
|
119
|
+
).run(proc.stdout, proc.stdin)
|
120
|
+
|
121
|
+
chan = Channel(proc.stdout, proc.stdin)
|
153
122
|
|
154
123
|
#
|
155
124
|
|
156
125
|
for ci in [
|
157
|
-
SubprocessCommand(
|
158
|
-
|
159
|
-
input=b'print(1)\n',
|
160
|
-
capture_stdout=True,
|
161
|
-
),
|
162
|
-
SubprocessCommand(
|
163
|
-
args=['uname'],
|
164
|
-
capture_stdout=True,
|
165
|
-
),
|
126
|
+
SubprocessCommand(['python3', '-'], input=b'print(1)\n'),
|
127
|
+
SubprocessCommand(['uname']),
|
166
128
|
]:
|
167
|
-
|
129
|
+
chan.send_obj(ci, Command)
|
168
130
|
|
169
|
-
o =
|
131
|
+
o = chan.recv_obj(Command.Output)
|
170
132
|
|
171
133
|
print(o)
|
172
134
|
|
@@ -0,0 +1,45 @@
|
|
1
|
+
import json
|
2
|
+
import struct
|
3
|
+
import typing as ta
|
4
|
+
|
5
|
+
from omlish.lite.json import json_dumps_compact
|
6
|
+
from omlish.lite.marshal import OBJ_MARSHALER_MANAGER
|
7
|
+
from omlish.lite.marshal import ObjMarshalerManager
|
8
|
+
|
9
|
+
|
10
|
+
class Channel:
|
11
|
+
def __init__(
|
12
|
+
self,
|
13
|
+
input: ta.IO, # noqa
|
14
|
+
output: ta.IO,
|
15
|
+
*,
|
16
|
+
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
17
|
+
) -> None:
|
18
|
+
super().__init__()
|
19
|
+
|
20
|
+
self._input = input
|
21
|
+
self._output = output
|
22
|
+
self._msh = msh
|
23
|
+
|
24
|
+
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
25
|
+
j = json_dumps_compact(self._msh.marshal_obj(o, ty))
|
26
|
+
d = j.encode('utf-8')
|
27
|
+
|
28
|
+
self._output.write(struct.pack('<I', len(d)))
|
29
|
+
self._output.write(d)
|
30
|
+
self._output.flush()
|
31
|
+
|
32
|
+
def recv_obj(self, ty: ta.Any) -> ta.Any:
|
33
|
+
d = self._input.read(4)
|
34
|
+
if not d:
|
35
|
+
return None
|
36
|
+
if len(d) != 4:
|
37
|
+
raise EOFError
|
38
|
+
|
39
|
+
sz = struct.unpack('<I', d)[0]
|
40
|
+
d = self._input.read(sz)
|
41
|
+
if len(d) != sz:
|
42
|
+
raise EOFError
|
43
|
+
|
44
|
+
j = json.loads(d.decode('utf-8'))
|
45
|
+
return self._msh.unmarshal_obj(j, ty)
|
@@ -6,6 +6,8 @@ import subprocess
|
|
6
6
|
import typing as ta
|
7
7
|
|
8
8
|
from omlish.lite.check import check_not_none
|
9
|
+
from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
|
10
|
+
from omlish.lite.subprocesses import SubprocessChannelOption
|
9
11
|
from omlish.lite.subprocesses import subprocess_maybe_shell_wrap_exec
|
10
12
|
|
11
13
|
|
@@ -19,7 +21,7 @@ class PySpawner:
|
|
19
21
|
shell: ta.Optional[str] = None,
|
20
22
|
shell_quote: bool = False,
|
21
23
|
python: str = DEFAULT_PYTHON,
|
22
|
-
stderr: ta.Optional[
|
24
|
+
stderr: ta.Optional[SubprocessChannelOption] = None,
|
23
25
|
) -> None:
|
24
26
|
super().__init__()
|
25
27
|
|
@@ -54,12 +56,6 @@ class PySpawner:
|
|
54
56
|
|
55
57
|
#
|
56
58
|
|
57
|
-
_STDERR_KWARG_MAP: ta.Mapping[str, int] = {
|
58
|
-
'pipe': subprocess.PIPE,
|
59
|
-
'stdout': subprocess.STDOUT,
|
60
|
-
'devnull': subprocess.DEVNULL,
|
61
|
-
}
|
62
|
-
|
63
59
|
@dc.dataclass(frozen=True)
|
64
60
|
class Spawned:
|
65
61
|
stdin: ta.IO
|
@@ -79,7 +75,7 @@ class PySpawner:
|
|
79
75
|
shell=pc.shell,
|
80
76
|
stdin=subprocess.PIPE,
|
81
77
|
stdout=subprocess.PIPE,
|
82
|
-
stderr=
|
78
|
+
stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[self._stderr] if self._stderr is not None else None,
|
83
79
|
) as proc:
|
84
80
|
stdin = check_not_none(proc.stdin)
|
85
81
|
stdout = check_not_none(proc.stdout)
|
@@ -177,12 +177,14 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
|
|
177
177
|
os.close(f)
|
178
178
|
|
179
179
|
# Save vars
|
180
|
-
os.environ
|
181
|
-
|
182
|
-
|
180
|
+
env = os.environ
|
181
|
+
exe = sys.executable
|
182
|
+
env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
|
183
|
+
env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
|
184
|
+
env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
|
183
185
|
|
184
186
|
# Start repl reading stdin from r0
|
185
|
-
os.execl(
|
187
|
+
os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
|
186
188
|
|
187
189
|
else:
|
188
190
|
# Child process
|
@@ -246,12 +248,12 @@ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
|
|
246
248
|
if cl.strip()
|
247
249
|
)
|
248
250
|
|
249
|
-
bs_z = zlib.compress(bs_src.encode('utf-8'))
|
250
|
-
|
251
|
+
bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
|
252
|
+
bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
|
251
253
|
|
252
254
|
stmts = [
|
253
255
|
f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
|
254
|
-
f'exec(zlib.decompress(base64.
|
256
|
+
f'exec(zlib.decompress(base64.b85decode({bs_z85!r})))',
|
255
257
|
f'_pyremote_bootstrap_main({context_name!r})',
|
256
258
|
]
|
257
259
|
|
@@ -440,7 +442,7 @@ class PyremoteBootstrapDriver:
|
|
440
442
|
|
441
443
|
#
|
442
444
|
|
443
|
-
def run(self,
|
445
|
+
def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
|
444
446
|
gen = self.gen()
|
445
447
|
|
446
448
|
gi: ta.Optional[bytes] = None
|
@@ -454,11 +456,11 @@ class PyremoteBootstrapDriver:
|
|
454
456
|
return e.value
|
455
457
|
|
456
458
|
if isinstance(go, self.Read):
|
457
|
-
if len(gi :=
|
459
|
+
if len(gi := input.read(go.sz)) != go.sz:
|
458
460
|
raise EOFError
|
459
461
|
elif isinstance(go, self.Write):
|
460
462
|
gi = None
|
461
|
-
|
462
|
-
|
463
|
+
output.write(go.d)
|
464
|
+
output.flush()
|
463
465
|
else:
|
464
466
|
raise TypeError(go)
|
@@ -72,6 +72,9 @@ ConfigMapping = ta.Mapping[str, ta.Any]
|
|
72
72
|
# ../../../threadworkers.py
|
73
73
|
ThreadWorkerT = ta.TypeVar('ThreadWorkerT', bound='ThreadWorker')
|
74
74
|
|
75
|
+
# ../../../../omlish/lite/subprocesses.py
|
76
|
+
SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']
|
77
|
+
|
75
78
|
|
76
79
|
########################################
|
77
80
|
# ../../../../../omdev/toml/parser.py
|
@@ -2382,120 +2385,144 @@ _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
|
|
2382
2385
|
}
|
2383
2386
|
|
2384
2387
|
|
2385
|
-
|
2386
|
-
ty: ta.Any,
|
2387
|
-
rec: ta.Callable[[ta.Any], ObjMarshaler],
|
2388
|
-
*,
|
2389
|
-
nonstrict_dataclasses: bool = False,
|
2390
|
-
) -> ObjMarshaler:
|
2391
|
-
if isinstance(ty, type):
|
2392
|
-
if abc.ABC in ty.__bases__:
|
2393
|
-
return PolymorphicObjMarshaler.of([ # type: ignore
|
2394
|
-
PolymorphicObjMarshaler.Impl(
|
2395
|
-
ity,
|
2396
|
-
ity.__qualname__,
|
2397
|
-
rec(ity),
|
2398
|
-
)
|
2399
|
-
for ity in deep_subclasses(ty)
|
2400
|
-
if abc.ABC not in ity.__bases__
|
2401
|
-
])
|
2402
|
-
|
2403
|
-
if issubclass(ty, enum.Enum):
|
2404
|
-
return EnumObjMarshaler(ty)
|
2405
|
-
|
2406
|
-
if dc.is_dataclass(ty):
|
2407
|
-
return DataclassObjMarshaler(
|
2408
|
-
ty,
|
2409
|
-
{f.name: rec(f.type) for f in dc.fields(ty)},
|
2410
|
-
nonstrict=nonstrict_dataclasses,
|
2411
|
-
)
|
2388
|
+
##
|
2412
2389
|
|
2413
|
-
if is_generic_alias(ty):
|
2414
|
-
try:
|
2415
|
-
mt = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES[ta.get_origin(ty)]
|
2416
|
-
except KeyError:
|
2417
|
-
pass
|
2418
|
-
else:
|
2419
|
-
k, v = ta.get_args(ty)
|
2420
|
-
return MappingObjMarshaler(mt, rec(k), rec(v))
|
2421
2390
|
|
2422
|
-
|
2423
|
-
|
2424
|
-
|
2425
|
-
|
2426
|
-
|
2427
|
-
[
|
2428
|
-
|
2391
|
+
class ObjMarshalerManager:
|
2392
|
+
def __init__(
|
2393
|
+
self,
|
2394
|
+
*,
|
2395
|
+
default_obj_marshalers: ta.Dict[ta.Any, ObjMarshaler] = _DEFAULT_OBJ_MARSHALERS, # noqa
|
2396
|
+
generic_mapping_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES, # noqa
|
2397
|
+
generic_iterable_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES, # noqa
|
2398
|
+
) -> None:
|
2399
|
+
super().__init__()
|
2429
2400
|
|
2430
|
-
|
2431
|
-
|
2401
|
+
self._obj_marshalers = dict(default_obj_marshalers)
|
2402
|
+
self._generic_mapping_types = generic_mapping_types
|
2403
|
+
self._generic_iterable_types = generic_iterable_types
|
2432
2404
|
|
2433
|
-
|
2405
|
+
self._lock = threading.RLock()
|
2406
|
+
self._marshalers: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
|
2407
|
+
self._proxies: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
|
2434
2408
|
|
2409
|
+
#
|
2435
2410
|
|
2436
|
-
|
2411
|
+
def make_obj_marshaler(
|
2412
|
+
self,
|
2413
|
+
ty: ta.Any,
|
2414
|
+
rec: ta.Callable[[ta.Any], ObjMarshaler],
|
2415
|
+
*,
|
2416
|
+
nonstrict_dataclasses: bool = False,
|
2417
|
+
) -> ObjMarshaler:
|
2418
|
+
if isinstance(ty, type):
|
2419
|
+
if abc.ABC in ty.__bases__:
|
2420
|
+
return PolymorphicObjMarshaler.of([ # type: ignore
|
2421
|
+
PolymorphicObjMarshaler.Impl(
|
2422
|
+
ity,
|
2423
|
+
ity.__qualname__,
|
2424
|
+
rec(ity),
|
2425
|
+
)
|
2426
|
+
for ity in deep_subclasses(ty)
|
2427
|
+
if abc.ABC not in ity.__bases__
|
2428
|
+
])
|
2429
|
+
|
2430
|
+
if issubclass(ty, enum.Enum):
|
2431
|
+
return EnumObjMarshaler(ty)
|
2432
|
+
|
2433
|
+
if dc.is_dataclass(ty):
|
2434
|
+
return DataclassObjMarshaler(
|
2435
|
+
ty,
|
2436
|
+
{f.name: rec(f.type) for f in dc.fields(ty)},
|
2437
|
+
nonstrict=nonstrict_dataclasses,
|
2438
|
+
)
|
2437
2439
|
|
2440
|
+
if is_generic_alias(ty):
|
2441
|
+
try:
|
2442
|
+
mt = self._generic_mapping_types[ta.get_origin(ty)]
|
2443
|
+
except KeyError:
|
2444
|
+
pass
|
2445
|
+
else:
|
2446
|
+
k, v = ta.get_args(ty)
|
2447
|
+
return MappingObjMarshaler(mt, rec(k), rec(v))
|
2438
2448
|
|
2439
|
-
|
2449
|
+
try:
|
2450
|
+
st = self._generic_iterable_types[ta.get_origin(ty)]
|
2451
|
+
except KeyError:
|
2452
|
+
pass
|
2453
|
+
else:
|
2454
|
+
[e] = ta.get_args(ty)
|
2455
|
+
return IterableObjMarshaler(st, rec(e))
|
2440
2456
|
|
2441
|
-
|
2457
|
+
if is_union_alias(ty):
|
2458
|
+
return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
|
2442
2459
|
|
2443
|
-
|
2460
|
+
raise TypeError(ty)
|
2444
2461
|
|
2462
|
+
#
|
2445
2463
|
|
2446
|
-
def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
|
2447
|
-
|
2448
|
-
|
2449
|
-
|
2450
|
-
|
2464
|
+
def register_opj_marshaler(self, ty: ta.Any, m: ObjMarshaler) -> None:
|
2465
|
+
with self._lock:
|
2466
|
+
if ty in self._obj_marshalers:
|
2467
|
+
raise KeyError(ty)
|
2468
|
+
self._obj_marshalers[ty] = m
|
2451
2469
|
|
2470
|
+
def get_obj_marshaler(
|
2471
|
+
self,
|
2472
|
+
ty: ta.Any,
|
2473
|
+
*,
|
2474
|
+
no_cache: bool = False,
|
2475
|
+
**kwargs: ta.Any,
|
2476
|
+
) -> ObjMarshaler:
|
2477
|
+
with self._lock:
|
2478
|
+
if not no_cache:
|
2479
|
+
try:
|
2480
|
+
return self._obj_marshalers[ty]
|
2481
|
+
except KeyError:
|
2482
|
+
pass
|
2452
2483
|
|
2453
|
-
def get_obj_marshaler(
|
2454
|
-
ty: ta.Any,
|
2455
|
-
*,
|
2456
|
-
no_cache: bool = False,
|
2457
|
-
**kwargs: ta.Any,
|
2458
|
-
) -> ObjMarshaler:
|
2459
|
-
with _OBJ_MARSHALERS_LOCK:
|
2460
|
-
if not no_cache:
|
2461
2484
|
try:
|
2462
|
-
return
|
2485
|
+
return self._proxies[ty]
|
2463
2486
|
except KeyError:
|
2464
2487
|
pass
|
2465
2488
|
|
2466
|
-
|
2467
|
-
|
2468
|
-
|
2469
|
-
|
2489
|
+
rec = functools.partial(
|
2490
|
+
self.get_obj_marshaler,
|
2491
|
+
no_cache=no_cache,
|
2492
|
+
**kwargs,
|
2493
|
+
)
|
2470
2494
|
|
2471
|
-
|
2472
|
-
|
2473
|
-
|
2474
|
-
|
2475
|
-
|
2495
|
+
p = ProxyObjMarshaler()
|
2496
|
+
self._proxies[ty] = p
|
2497
|
+
try:
|
2498
|
+
m = self.make_obj_marshaler(ty, rec, **kwargs)
|
2499
|
+
finally:
|
2500
|
+
del self._proxies[ty]
|
2501
|
+
p.m = m
|
2476
2502
|
|
2477
|
-
|
2478
|
-
|
2479
|
-
|
2480
|
-
|
2481
|
-
|
2482
|
-
del _OBJ_MARSHALER_PROXIES[ty]
|
2483
|
-
p.m = m
|
2503
|
+
if not no_cache:
|
2504
|
+
self._obj_marshalers[ty] = m
|
2505
|
+
return m
|
2506
|
+
|
2507
|
+
#
|
2484
2508
|
|
2485
|
-
|
2486
|
-
|
2487
|
-
|
2509
|
+
def marshal_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Any:
|
2510
|
+
return self.get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
|
2511
|
+
|
2512
|
+
def unmarshal_obj(self, o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
|
2513
|
+
return self.get_obj_marshaler(ty).unmarshal(o)
|
2488
2514
|
|
2489
2515
|
|
2490
2516
|
##
|
2491
2517
|
|
2492
2518
|
|
2493
|
-
|
2494
|
-
return get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
|
2519
|
+
OBJ_MARSHALER_MANAGER = ObjMarshalerManager()
|
2495
2520
|
|
2521
|
+
register_opj_marshaler = OBJ_MARSHALER_MANAGER.register_opj_marshaler
|
2522
|
+
get_obj_marshaler = OBJ_MARSHALER_MANAGER.get_obj_marshaler
|
2496
2523
|
|
2497
|
-
|
2498
|
-
|
2524
|
+
marshal_obj = OBJ_MARSHALER_MANAGER.marshal_obj
|
2525
|
+
unmarshal_obj = OBJ_MARSHALER_MANAGER.unmarshal_obj
|
2499
2526
|
|
2500
2527
|
|
2501
2528
|
########################################
|
@@ -3058,6 +3085,16 @@ class ThreadWorkerGroup:
|
|
3058
3085
|
##
|
3059
3086
|
|
3060
3087
|
|
3088
|
+
SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
|
3089
|
+
'pipe': subprocess.PIPE,
|
3090
|
+
'stdout': subprocess.STDOUT,
|
3091
|
+
'devnull': subprocess.DEVNULL,
|
3092
|
+
}
|
3093
|
+
|
3094
|
+
|
3095
|
+
##
|
3096
|
+
|
3097
|
+
|
3061
3098
|
_SUBPROCESS_SHELL_WRAP_EXECS = False
|
3062
3099
|
|
3063
3100
|
|