ominfra 0.0.0.dev147__py3-none-any.whl → 0.0.0.dev148__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- ominfra/configs.py +30 -5
- ominfra/manage/commands/base.py +5 -5
- ominfra/manage/commands/execution.py +2 -2
- ominfra/manage/commands/inject.py +1 -1
- ominfra/manage/commands/interp.py +2 -2
- ominfra/manage/commands/subprocess.py +22 -14
- ominfra/manage/deploy/command.py +1 -1
- ominfra/manage/main.py +48 -32
- ominfra/manage/remote/_main.py +172 -0
- ominfra/manage/remote/channel.py +41 -16
- ominfra/manage/remote/config.py +10 -0
- ominfra/manage/remote/connection.py +106 -0
- ominfra/manage/remote/execution.py +244 -155
- ominfra/manage/remote/inject.py +7 -3
- ominfra/manage/remote/spawning.py +51 -33
- ominfra/pyremote.py +28 -3
- ominfra/scripts/journald2aws.py +127 -28
- ominfra/scripts/manage.py +1360 -485
- ominfra/scripts/supervisor.py +59 -10
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/RECORD +25 -23
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -3,16 +3,20 @@
|
|
3
3
|
# @omlish-lite
|
4
4
|
# @omlish-script
|
5
5
|
# @omlish-amalg-output ../manage/main.py
|
6
|
-
# ruff: noqa: N802 UP006 UP007 UP036
|
6
|
+
# ruff: noqa: N802 TC003 UP006 UP007 UP036
|
7
7
|
"""
|
8
8
|
manage.py -s 'docker run -i python:3.12'
|
9
9
|
manage.py -s 'ssh -i /foo/bar.pem foo@bar.baz' -q --python=python3.8
|
10
10
|
"""
|
11
11
|
import abc
|
12
|
+
import asyncio
|
13
|
+
import asyncio.base_subprocess
|
14
|
+
import asyncio.subprocess
|
12
15
|
import base64
|
13
16
|
import collections
|
14
17
|
import collections.abc
|
15
18
|
import contextlib
|
19
|
+
import ctypes as ct
|
16
20
|
import dataclasses as dc
|
17
21
|
import datetime
|
18
22
|
import decimal
|
@@ -30,6 +34,7 @@ import pwd
|
|
30
34
|
import re
|
31
35
|
import shlex
|
32
36
|
import shutil
|
37
|
+
import signal
|
33
38
|
import site
|
34
39
|
import struct
|
35
40
|
import subprocess
|
@@ -62,6 +67,9 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
|
|
62
67
|
VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType] # noqa
|
63
68
|
VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
|
64
69
|
|
70
|
+
# ../../omlish/lite/asyncio/asyncio.py
|
71
|
+
AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
|
72
|
+
|
65
73
|
# ../../omlish/lite/cached.py
|
66
74
|
T = ta.TypeVar('T')
|
67
75
|
CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
|
@@ -822,10 +830,8 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
822
830
|
with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
|
823
831
|
main_src = sf.read()
|
824
832
|
|
825
|
-
# Restore
|
833
|
+
# Restore vars
|
826
834
|
sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
|
827
|
-
|
828
|
-
# Grab context name
|
829
835
|
context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
|
830
836
|
|
831
837
|
# Write third ack
|
@@ -1000,12 +1006,102 @@ class PyremoteBootstrapDriver:
|
|
1000
1006
|
else:
|
1001
1007
|
raise TypeError(go)
|
1002
1008
|
|
1009
|
+
async def async_run(
|
1010
|
+
self,
|
1011
|
+
input: ta.Any, # asyncio.StreamWriter # noqa
|
1012
|
+
output: ta.Any, # asyncio.StreamReader
|
1013
|
+
) -> Result:
|
1014
|
+
gen = self.gen()
|
1015
|
+
|
1016
|
+
gi: ta.Optional[bytes] = None
|
1017
|
+
while True:
|
1018
|
+
try:
|
1019
|
+
if gi is not None:
|
1020
|
+
go = gen.send(gi)
|
1021
|
+
else:
|
1022
|
+
go = next(gen)
|
1023
|
+
except StopIteration as e:
|
1024
|
+
return e.value
|
1025
|
+
|
1026
|
+
if isinstance(go, self.Read):
|
1027
|
+
if len(gi := await input.read(go.sz)) != go.sz:
|
1028
|
+
raise EOFError
|
1029
|
+
elif isinstance(go, self.Write):
|
1030
|
+
gi = None
|
1031
|
+
output.write(go.d)
|
1032
|
+
await output.drain()
|
1033
|
+
else:
|
1034
|
+
raise TypeError(go)
|
1035
|
+
|
1036
|
+
|
1037
|
+
########################################
|
1038
|
+
# ../../../omlish/lite/asyncio/asyncio.py
|
1039
|
+
|
1040
|
+
|
1041
|
+
##
|
1042
|
+
|
1043
|
+
|
1044
|
+
ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
|
1045
|
+
|
1046
|
+
|
1047
|
+
async def asyncio_open_stream_reader(
|
1048
|
+
f: ta.IO,
|
1049
|
+
loop: ta.Any = None,
|
1050
|
+
*,
|
1051
|
+
limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
|
1052
|
+
) -> asyncio.StreamReader:
|
1053
|
+
if loop is None:
|
1054
|
+
loop = asyncio.get_running_loop()
|
1055
|
+
|
1056
|
+
reader = asyncio.StreamReader(limit=limit, loop=loop)
|
1057
|
+
await loop.connect_read_pipe(
|
1058
|
+
lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
|
1059
|
+
f,
|
1060
|
+
)
|
1061
|
+
|
1062
|
+
return reader
|
1063
|
+
|
1064
|
+
|
1065
|
+
async def asyncio_open_stream_writer(
|
1066
|
+
f: ta.IO,
|
1067
|
+
loop: ta.Any = None,
|
1068
|
+
) -> asyncio.StreamWriter:
|
1069
|
+
if loop is None:
|
1070
|
+
loop = asyncio.get_running_loop()
|
1071
|
+
|
1072
|
+
writer_transport, writer_protocol = await loop.connect_write_pipe(
|
1073
|
+
lambda: asyncio.streams.FlowControlMixin(loop=loop),
|
1074
|
+
f,
|
1075
|
+
)
|
1076
|
+
|
1077
|
+
return asyncio.streams.StreamWriter(
|
1078
|
+
writer_transport,
|
1079
|
+
writer_protocol,
|
1080
|
+
None,
|
1081
|
+
loop,
|
1082
|
+
)
|
1083
|
+
|
1084
|
+
|
1085
|
+
##
|
1086
|
+
|
1087
|
+
|
1088
|
+
def asyncio_maybe_timeout(
|
1089
|
+
fut: AwaitableT,
|
1090
|
+
timeout: ta.Optional[float] = None,
|
1091
|
+
) -> AwaitableT:
|
1092
|
+
if timeout is not None:
|
1093
|
+
fut = asyncio.wait_for(fut, timeout) # type: ignore
|
1094
|
+
return fut
|
1095
|
+
|
1003
1096
|
|
1004
1097
|
########################################
|
1005
1098
|
# ../../../omlish/lite/cached.py
|
1006
1099
|
|
1007
1100
|
|
1008
|
-
|
1101
|
+
##
|
1102
|
+
|
1103
|
+
|
1104
|
+
class _AbstractCachedNullary:
|
1009
1105
|
def __init__(self, fn):
|
1010
1106
|
super().__init__()
|
1011
1107
|
self._fn = fn
|
@@ -1013,17 +1109,25 @@ class _cached_nullary: # noqa
|
|
1013
1109
|
functools.update_wrapper(self, fn)
|
1014
1110
|
|
1015
1111
|
def __call__(self, *args, **kwargs): # noqa
|
1016
|
-
|
1017
|
-
self._value = self._fn()
|
1018
|
-
return self._value
|
1112
|
+
raise TypeError
|
1019
1113
|
|
1020
1114
|
def __get__(self, instance, owner): # noqa
|
1021
1115
|
bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
|
1022
1116
|
return bound
|
1023
1117
|
|
1024
1118
|
|
1119
|
+
##
|
1120
|
+
|
1121
|
+
|
1122
|
+
class _CachedNullary(_AbstractCachedNullary):
|
1123
|
+
def __call__(self, *args, **kwargs): # noqa
|
1124
|
+
if self._value is self._missing:
|
1125
|
+
self._value = self._fn()
|
1126
|
+
return self._value
|
1127
|
+
|
1128
|
+
|
1025
1129
|
def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
1026
|
-
return
|
1130
|
+
return _CachedNullary(fn)
|
1027
1131
|
|
1028
1132
|
|
1029
1133
|
def static_init(fn: CallableT) -> CallableT:
|
@@ -1032,6 +1136,20 @@ def static_init(fn: CallableT) -> CallableT:
|
|
1032
1136
|
return fn
|
1033
1137
|
|
1034
1138
|
|
1139
|
+
##
|
1140
|
+
|
1141
|
+
|
1142
|
+
class _AsyncCachedNullary(_AbstractCachedNullary):
|
1143
|
+
async def __call__(self, *args, **kwargs):
|
1144
|
+
if self._value is self._missing:
|
1145
|
+
self._value = await self._fn()
|
1146
|
+
return self._value
|
1147
|
+
|
1148
|
+
|
1149
|
+
def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
1150
|
+
return _AsyncCachedNullary(fn)
|
1151
|
+
|
1152
|
+
|
1035
1153
|
########################################
|
1036
1154
|
# ../../../omlish/lite/check.py
|
1037
1155
|
|
@@ -1129,6 +1247,30 @@ def check_non_empty(v: SizedT) -> SizedT:
|
|
1129
1247
|
return v
|
1130
1248
|
|
1131
1249
|
|
1250
|
+
########################################
|
1251
|
+
# ../../../omlish/lite/deathsig.py
|
1252
|
+
|
1253
|
+
|
1254
|
+
LINUX_PR_SET_PDEATHSIG = 1 # Second arg is a signal
|
1255
|
+
LINUX_PR_GET_PDEATHSIG = 2 # Second arg is a ptr to return the signal
|
1256
|
+
|
1257
|
+
|
1258
|
+
def set_process_deathsig(sig: int) -> bool:
|
1259
|
+
if sys.platform == 'linux':
|
1260
|
+
libc = ct.CDLL('libc.so.6')
|
1261
|
+
|
1262
|
+
# int prctl(int option, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5);
|
1263
|
+
libc.prctl.restype = ct.c_int
|
1264
|
+
libc.prctl.argtypes = [ct.c_int, ct.c_ulong, ct.c_ulong, ct.c_ulong, ct.c_ulong]
|
1265
|
+
|
1266
|
+
libc.prctl(LINUX_PR_SET_PDEATHSIG, sig, 0, 0, 0, 0)
|
1267
|
+
|
1268
|
+
return True
|
1269
|
+
|
1270
|
+
else:
|
1271
|
+
return False
|
1272
|
+
|
1273
|
+
|
1132
1274
|
########################################
|
1133
1275
|
# ../../../omlish/lite/json.py
|
1134
1276
|
|
@@ -1910,8 +2052,8 @@ class Command(abc.ABC, ta.Generic[CommandOutputT]):
|
|
1910
2052
|
pass
|
1911
2053
|
|
1912
2054
|
@ta.final
|
1913
|
-
def execute(self, executor: 'CommandExecutor') -> CommandOutputT:
|
1914
|
-
return check_isinstance(executor.execute(self), self.Output) # type: ignore[return-value]
|
2055
|
+
async def execute(self, executor: 'CommandExecutor') -> CommandOutputT:
|
2056
|
+
return check_isinstance(await executor.execute(self), self.Output) # type: ignore[return-value]
|
1915
2057
|
|
1916
2058
|
|
1917
2059
|
##
|
@@ -1972,10 +2114,10 @@ class CommandOutputOrExceptionData(CommandOutputOrException):
|
|
1972
2114
|
|
1973
2115
|
class CommandExecutor(abc.ABC, ta.Generic[CommandT, CommandOutputT]):
|
1974
2116
|
@abc.abstractmethod
|
1975
|
-
def execute(self, cmd: CommandT) -> CommandOutputT:
|
2117
|
+
def execute(self, cmd: CommandT) -> ta.Awaitable[CommandOutputT]:
|
1976
2118
|
raise NotImplementedError
|
1977
2119
|
|
1978
|
-
def try_execute(
|
2120
|
+
async def try_execute(
|
1979
2121
|
self,
|
1980
2122
|
cmd: CommandT,
|
1981
2123
|
*,
|
@@ -1983,7 +2125,7 @@ class CommandExecutor(abc.ABC, ta.Generic[CommandT, CommandOutputT]):
|
|
1983
2125
|
omit_exc_object: bool = False,
|
1984
2126
|
) -> CommandOutputOrException[CommandOutputT]:
|
1985
2127
|
try:
|
1986
|
-
o = self.execute(cmd)
|
2128
|
+
o = await self.execute(cmd)
|
1987
2129
|
|
1988
2130
|
except Exception as e: # noqa
|
1989
2131
|
if log is not None:
|
@@ -2054,8 +2196,18 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
|
|
2054
2196
|
class RemoteConfig:
|
2055
2197
|
payload_file: ta.Optional[str] = None
|
2056
2198
|
|
2199
|
+
set_pgid: bool = True
|
2200
|
+
|
2201
|
+
deathsig: ta.Optional[str] = 'KILL'
|
2202
|
+
|
2057
2203
|
pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
|
2058
2204
|
|
2205
|
+
forward_logging: bool = True
|
2206
|
+
|
2207
|
+
timebomb_delay_s: ta.Optional[float] = 60 * 60.
|
2208
|
+
|
2209
|
+
heartbeat_interval_s: float = 3.
|
2210
|
+
|
2059
2211
|
|
2060
2212
|
########################################
|
2061
2213
|
# ../remote/payload.py
|
@@ -3788,6 +3940,8 @@ class InterpSpecifier:
|
|
3788
3940
|
s, o = InterpOpts.parse_suffix(s)
|
3789
3941
|
if not any(s.startswith(o) for o in Specifier.OPERATORS):
|
3790
3942
|
s = '~=' + s
|
3943
|
+
if s.count('.') < 2:
|
3944
|
+
s += '.0'
|
3791
3945
|
return cls(
|
3792
3946
|
specifier=Specifier(s),
|
3793
3947
|
opts=o,
|
@@ -3834,9 +3988,9 @@ class LocalCommandExecutor(CommandExecutor):
|
|
3834
3988
|
|
3835
3989
|
self._command_executors = command_executors
|
3836
3990
|
|
3837
|
-
def execute(self, cmd: Command) -> Command.Output:
|
3991
|
+
async def execute(self, cmd: Command) -> Command.Output:
|
3838
3992
|
ce: CommandExecutor = self._command_executors[type(cmd)]
|
3839
|
-
return ce.execute(cmd)
|
3993
|
+
return await ce.execute(cmd)
|
3840
3994
|
|
3841
3995
|
|
3842
3996
|
########################################
|
@@ -3882,7 +4036,7 @@ class DeployCommand(Command['DeployCommand.Output']):
|
|
3882
4036
|
|
3883
4037
|
|
3884
4038
|
class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
|
3885
|
-
def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
4039
|
+
async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
3886
4040
|
log.info('Deploying!')
|
3887
4041
|
|
3888
4042
|
return DeployCommand.Output()
|
@@ -3904,11 +4058,30 @@ ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMar
|
|
3904
4058
|
# ../remote/channel.py
|
3905
4059
|
|
3906
4060
|
|
3907
|
-
|
4061
|
+
##
|
4062
|
+
|
4063
|
+
|
4064
|
+
class RemoteChannel(abc.ABC):
|
4065
|
+
@abc.abstractmethod
|
4066
|
+
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Awaitable[None]:
|
4067
|
+
raise NotImplementedError
|
4068
|
+
|
4069
|
+
@abc.abstractmethod
|
4070
|
+
def recv_obj(self, ty: ta.Type[T]) -> ta.Awaitable[ta.Optional[T]]:
|
4071
|
+
raise NotImplementedError
|
4072
|
+
|
4073
|
+
def set_marshaler(self, msh: ObjMarshalerManager) -> None: # noqa
|
4074
|
+
pass
|
4075
|
+
|
4076
|
+
|
4077
|
+
##
|
4078
|
+
|
4079
|
+
|
4080
|
+
class RemoteChannelImpl(RemoteChannel):
|
3908
4081
|
def __init__(
|
3909
4082
|
self,
|
3910
|
-
input:
|
3911
|
-
output:
|
4083
|
+
input: asyncio.StreamReader, # noqa
|
4084
|
+
output: asyncio.StreamWriter,
|
3912
4085
|
*,
|
3913
4086
|
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
3914
4087
|
) -> None:
|
@@ -3918,41 +4091,46 @@ class RemoteChannel:
|
|
3918
4091
|
self._output = output
|
3919
4092
|
self._msh = msh
|
3920
4093
|
|
3921
|
-
self.
|
4094
|
+
self._input_lock = asyncio.Lock()
|
4095
|
+
self._output_lock = asyncio.Lock()
|
3922
4096
|
|
3923
4097
|
def set_marshaler(self, msh: ObjMarshalerManager) -> None:
|
3924
4098
|
self._msh = msh
|
3925
4099
|
|
3926
|
-
|
4100
|
+
#
|
4101
|
+
|
4102
|
+
async def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
3927
4103
|
j = json_dumps_compact(self._msh.marshal_obj(o, ty))
|
3928
4104
|
d = j.encode('utf-8')
|
3929
4105
|
|
3930
4106
|
self._output.write(struct.pack('<I', len(d)))
|
3931
4107
|
self._output.write(d)
|
3932
|
-
self._output.
|
4108
|
+
await self._output.drain()
|
3933
4109
|
|
3934
|
-
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
3935
|
-
with self.
|
3936
|
-
return self._send_obj(o, ty)
|
4110
|
+
async def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
4111
|
+
async with self._output_lock:
|
4112
|
+
return await self._send_obj(o, ty)
|
4113
|
+
|
4114
|
+
#
|
3937
4115
|
|
3938
|
-
def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
3939
|
-
d = self._input.read(4)
|
4116
|
+
async def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
4117
|
+
d = await self._input.read(4)
|
3940
4118
|
if not d:
|
3941
4119
|
return None
|
3942
4120
|
if len(d) != 4:
|
3943
4121
|
raise EOFError
|
3944
4122
|
|
3945
4123
|
sz = struct.unpack('<I', d)[0]
|
3946
|
-
d = self._input.read(sz)
|
4124
|
+
d = await self._input.read(sz)
|
3947
4125
|
if len(d) != sz:
|
3948
4126
|
raise EOFError
|
3949
4127
|
|
3950
4128
|
j = json.loads(d.decode('utf-8'))
|
3951
4129
|
return self._msh.unmarshal_obj(j, ty)
|
3952
4130
|
|
3953
|
-
def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
3954
|
-
with self.
|
3955
|
-
return self._recv_obj(ty)
|
4131
|
+
async def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
4132
|
+
async with self._input_lock:
|
4133
|
+
return await self._recv_obj(ty)
|
3956
4134
|
|
3957
4135
|
|
3958
4136
|
########################################
|
@@ -3986,7 +4164,7 @@ def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
|
|
3986
4164
|
return args
|
3987
4165
|
|
3988
4166
|
|
3989
|
-
def
|
4167
|
+
def prepare_subprocess_invocation(
|
3990
4168
|
*args: str,
|
3991
4169
|
env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
3992
4170
|
extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
@@ -3994,9 +4172,9 @@ def _prepare_subprocess_invocation(
|
|
3994
4172
|
shell: bool = False,
|
3995
4173
|
**kwargs: ta.Any,
|
3996
4174
|
) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
|
3997
|
-
log.debug(args)
|
4175
|
+
log.debug('prepare_subprocess_invocation: args=%r', args)
|
3998
4176
|
if extra_env:
|
3999
|
-
log.debug(extra_env)
|
4177
|
+
log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
|
4000
4178
|
|
4001
4179
|
if extra_env:
|
4002
4180
|
env = {**(env if env is not None else os.environ), **extra_env}
|
@@ -4015,14 +4193,46 @@ def _prepare_subprocess_invocation(
|
|
4015
4193
|
)
|
4016
4194
|
|
4017
4195
|
|
4018
|
-
|
4019
|
-
|
4020
|
-
|
4196
|
+
##
|
4197
|
+
|
4198
|
+
|
4199
|
+
@contextlib.contextmanager
|
4200
|
+
def subprocess_common_context(*args: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
|
4201
|
+
start_time = time.time()
|
4202
|
+
try:
|
4203
|
+
log.debug('subprocess_common_context.try: args=%r', args)
|
4204
|
+
yield
|
4205
|
+
|
4206
|
+
except Exception as exc: # noqa
|
4207
|
+
log.debug('subprocess_common_context.except: exc=%r', exc)
|
4208
|
+
raise
|
4209
|
+
|
4210
|
+
finally:
|
4211
|
+
end_time = time.time()
|
4212
|
+
elapsed_s = end_time - start_time
|
4213
|
+
log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
|
4214
|
+
|
4215
|
+
|
4216
|
+
##
|
4217
|
+
|
4218
|
+
|
4219
|
+
def subprocess_check_call(
|
4220
|
+
*args: str,
|
4221
|
+
stdout: ta.Any = sys.stderr,
|
4222
|
+
**kwargs: ta.Any,
|
4223
|
+
) -> None:
|
4224
|
+
args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
|
4225
|
+
with subprocess_common_context(*args, **kwargs):
|
4226
|
+
return subprocess.check_call(args, **kwargs) # type: ignore
|
4021
4227
|
|
4022
4228
|
|
4023
|
-
def subprocess_check_output(
|
4024
|
-
|
4025
|
-
|
4229
|
+
def subprocess_check_output(
|
4230
|
+
*args: str,
|
4231
|
+
**kwargs: ta.Any,
|
4232
|
+
) -> bytes:
|
4233
|
+
args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
|
4234
|
+
with subprocess_common_context(*args, **kwargs):
|
4235
|
+
return subprocess.check_output(args, **kwargs)
|
4026
4236
|
|
4027
4237
|
|
4028
4238
|
def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
|
@@ -4038,16 +4248,31 @@ DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
|
|
4038
4248
|
)
|
4039
4249
|
|
4040
4250
|
|
4041
|
-
def
|
4042
|
-
|
4251
|
+
def _subprocess_try_run(
|
4252
|
+
fn: ta.Callable[..., T],
|
4253
|
+
*args: ta.Any,
|
4043
4254
|
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4044
4255
|
**kwargs: ta.Any,
|
4045
|
-
) ->
|
4256
|
+
) -> ta.Union[T, Exception]:
|
4046
4257
|
try:
|
4047
|
-
|
4258
|
+
return fn(*args, **kwargs)
|
4048
4259
|
except try_exceptions as e: # noqa
|
4049
4260
|
if log.isEnabledFor(logging.DEBUG):
|
4050
4261
|
log.exception('command failed')
|
4262
|
+
return e
|
4263
|
+
|
4264
|
+
|
4265
|
+
def subprocess_try_call(
|
4266
|
+
*args: str,
|
4267
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4268
|
+
**kwargs: ta.Any,
|
4269
|
+
) -> bool:
|
4270
|
+
if isinstance(_subprocess_try_run(
|
4271
|
+
subprocess_check_call,
|
4272
|
+
*args,
|
4273
|
+
try_exceptions=try_exceptions,
|
4274
|
+
**kwargs,
|
4275
|
+
), Exception):
|
4051
4276
|
return False
|
4052
4277
|
else:
|
4053
4278
|
return True
|
@@ -4058,12 +4283,15 @@ def subprocess_try_output(
|
|
4058
4283
|
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4059
4284
|
**kwargs: ta.Any,
|
4060
4285
|
) -> ta.Optional[bytes]:
|
4061
|
-
|
4062
|
-
|
4063
|
-
|
4064
|
-
|
4065
|
-
|
4286
|
+
if isinstance(ret := _subprocess_try_run(
|
4287
|
+
subprocess_check_output,
|
4288
|
+
*args,
|
4289
|
+
try_exceptions=try_exceptions,
|
4290
|
+
**kwargs,
|
4291
|
+
), Exception):
|
4066
4292
|
return None
|
4293
|
+
else:
|
4294
|
+
return ret
|
4067
4295
|
|
4068
4296
|
|
4069
4297
|
def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|
@@ -4090,175 +4318,922 @@ def subprocess_close(
|
|
4090
4318
|
|
4091
4319
|
|
4092
4320
|
########################################
|
4093
|
-
#
|
4321
|
+
# ../remote/execution.py
|
4094
4322
|
|
4095
4323
|
|
4096
|
-
|
4097
|
-
class InterpInspection:
|
4098
|
-
exe: str
|
4099
|
-
version: Version
|
4324
|
+
##
|
4100
4325
|
|
4101
|
-
version_str: str
|
4102
|
-
config_vars: ta.Mapping[str, str]
|
4103
|
-
prefix: str
|
4104
|
-
base_prefix: str
|
4105
4326
|
|
4106
|
-
|
4107
|
-
|
4108
|
-
|
4109
|
-
|
4110
|
-
debug=bool(self.config_vars.get('Py_DEBUG')),
|
4111
|
-
)
|
4327
|
+
class _RemoteProtocol:
|
4328
|
+
class Message(abc.ABC): # noqa
|
4329
|
+
async def send(self, chan: RemoteChannel) -> None:
|
4330
|
+
await chan.send_obj(self, _RemoteProtocol.Message)
|
4112
4331
|
|
4113
|
-
|
4114
|
-
|
4115
|
-
|
4116
|
-
version=self.version,
|
4117
|
-
opts=self.opts,
|
4118
|
-
)
|
4332
|
+
@classmethod
|
4333
|
+
async def recv(cls: ta.Type[T], chan: RemoteChannel) -> ta.Optional[T]:
|
4334
|
+
return await chan.recv_obj(cls)
|
4119
4335
|
|
4120
|
-
|
4121
|
-
def is_venv(self) -> bool:
|
4122
|
-
return self.prefix != self.base_prefix
|
4336
|
+
#
|
4123
4337
|
|
4338
|
+
class Request(Message, abc.ABC): # noqa
|
4339
|
+
pass
|
4124
4340
|
|
4125
|
-
|
4341
|
+
@dc.dataclass(frozen=True)
|
4342
|
+
class CommandRequest(Request):
|
4343
|
+
seq: int
|
4344
|
+
cmd: Command
|
4126
4345
|
|
4127
|
-
|
4128
|
-
|
4346
|
+
@dc.dataclass(frozen=True)
|
4347
|
+
class PingRequest(Request):
|
4348
|
+
time: float
|
4129
4349
|
|
4130
|
-
|
4350
|
+
#
|
4131
4351
|
|
4132
|
-
|
4133
|
-
|
4134
|
-
version_str=__import__('sys').version,
|
4135
|
-
prefix=__import__('sys').prefix,
|
4136
|
-
base_prefix=__import__('sys').base_prefix,
|
4137
|
-
config_vars=__import__('sysconfig').get_config_vars(),
|
4138
|
-
))"""
|
4352
|
+
class Response(Message, abc.ABC): # noqa
|
4353
|
+
pass
|
4139
4354
|
|
4140
|
-
|
4355
|
+
@dc.dataclass(frozen=True)
|
4356
|
+
class LogResponse(Response):
|
4357
|
+
s: str
|
4141
4358
|
|
4142
|
-
@
|
4143
|
-
|
4144
|
-
|
4145
|
-
|
4146
|
-
) -> InterpInspection:
|
4147
|
-
dct = json.loads(output)
|
4359
|
+
@dc.dataclass(frozen=True)
|
4360
|
+
class CommandResponse(Response):
|
4361
|
+
seq: int
|
4362
|
+
res: CommandOutputOrExceptionData
|
4148
4363
|
|
4149
|
-
|
4364
|
+
@dc.dataclass(frozen=True)
|
4365
|
+
class PingResponse(Response):
|
4366
|
+
time: float
|
4150
4367
|
|
4151
|
-
return InterpInspection(
|
4152
|
-
exe=exe,
|
4153
|
-
version=version,
|
4154
|
-
**{k: dct[k] for k in (
|
4155
|
-
'version_str',
|
4156
|
-
'prefix',
|
4157
|
-
'base_prefix',
|
4158
|
-
'config_vars',
|
4159
|
-
)},
|
4160
|
-
)
|
4161
4368
|
|
4162
|
-
|
4163
|
-
def running(cls) -> 'InterpInspection':
|
4164
|
-
return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
|
4369
|
+
##
|
4165
4370
|
|
4166
|
-
def _inspect(self, exe: str) -> InterpInspection:
|
4167
|
-
output = subprocess_check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
|
4168
|
-
return self._build_inspection(exe, output.decode())
|
4169
4371
|
|
4170
|
-
|
4171
|
-
|
4172
|
-
|
4173
|
-
|
4174
|
-
|
4175
|
-
|
4176
|
-
|
4177
|
-
except Exception as e: # noqa
|
4178
|
-
if log.isEnabledFor(logging.DEBUG):
|
4179
|
-
log.exception('Failed to inspect interp: %s', exe)
|
4180
|
-
ret = None
|
4181
|
-
self._cache[exe] = ret
|
4182
|
-
return ret
|
4372
|
+
class _RemoteLogHandler(logging.Handler):
|
4373
|
+
def __init__(
|
4374
|
+
self,
|
4375
|
+
chan: RemoteChannel,
|
4376
|
+
loop: ta.Any = None,
|
4377
|
+
) -> None:
|
4378
|
+
super().__init__()
|
4183
4379
|
|
4380
|
+
self._chan = chan
|
4381
|
+
self._loop = loop
|
4184
4382
|
|
4185
|
-
|
4383
|
+
def emit(self, record):
|
4384
|
+
msg = self.format(record)
|
4186
4385
|
|
4386
|
+
async def inner():
|
4387
|
+
await _RemoteProtocol.LogResponse(msg).send(self._chan)
|
4187
4388
|
|
4188
|
-
|
4189
|
-
|
4389
|
+
loop = self._loop
|
4390
|
+
if loop is None:
|
4391
|
+
loop = asyncio.get_running_loop()
|
4392
|
+
if loop is not None:
|
4393
|
+
asyncio.run_coroutine_threadsafe(inner(), loop)
|
4190
4394
|
|
4191
4395
|
|
4192
4396
|
##
|
4193
4397
|
|
4194
4398
|
|
4195
|
-
|
4196
|
-
|
4197
|
-
|
4399
|
+
class _RemoteCommandHandler:
|
4400
|
+
def __init__(
|
4401
|
+
self,
|
4402
|
+
chan: RemoteChannel,
|
4403
|
+
executor: CommandExecutor,
|
4404
|
+
*,
|
4405
|
+
stop: ta.Optional[asyncio.Event] = None,
|
4406
|
+
) -> None:
|
4407
|
+
super().__init__()
|
4198
4408
|
|
4199
|
-
|
4200
|
-
|
4201
|
-
|
4409
|
+
self._chan = chan
|
4410
|
+
self._executor = executor
|
4411
|
+
self._stop = stop if stop is not None else asyncio.Event()
|
4202
4412
|
|
4203
|
-
|
4204
|
-
stderr: str = 'pipe' # SubprocessChannelOption
|
4413
|
+
self._cmds_by_seq: ta.Dict[int, _RemoteCommandHandler._Command] = {}
|
4205
4414
|
|
4206
|
-
|
4207
|
-
|
4415
|
+
@dc.dataclass(frozen=True)
|
4416
|
+
class _Command:
|
4417
|
+
req: _RemoteProtocol.CommandRequest
|
4418
|
+
fut: asyncio.Future
|
4419
|
+
|
4420
|
+
async def run(self) -> None:
|
4421
|
+
stop_task = asyncio.create_task(self._stop.wait())
|
4422
|
+
recv_task: ta.Optional[asyncio.Task] = None
|
4423
|
+
|
4424
|
+
while not self._stop.is_set():
|
4425
|
+
if recv_task is None:
|
4426
|
+
recv_task = asyncio.create_task(_RemoteProtocol.Request.recv(self._chan))
|
4427
|
+
|
4428
|
+
done, pending = await asyncio.wait([
|
4429
|
+
stop_task,
|
4430
|
+
recv_task,
|
4431
|
+
], return_when=asyncio.FIRST_COMPLETED)
|
4432
|
+
|
4433
|
+
if recv_task in done:
|
4434
|
+
msg: ta.Optional[_RemoteProtocol.Message] = check_isinstance(
|
4435
|
+
recv_task.result(),
|
4436
|
+
(_RemoteProtocol.Message, type(None)),
|
4437
|
+
)
|
4438
|
+
recv_task = None
|
4439
|
+
|
4440
|
+
if msg is None:
|
4441
|
+
break
|
4442
|
+
|
4443
|
+
await self._handle_message(msg)
|
4444
|
+
|
4445
|
+
async def _handle_message(self, msg: _RemoteProtocol.Message) -> None:
|
4446
|
+
if isinstance(msg, _RemoteProtocol.PingRequest):
|
4447
|
+
log.debug('Ping: %r', msg)
|
4448
|
+
await _RemoteProtocol.PingResponse(
|
4449
|
+
time=msg.time,
|
4450
|
+
).send(self._chan)
|
4451
|
+
|
4452
|
+
elif isinstance(msg, _RemoteProtocol.CommandRequest):
|
4453
|
+
fut = asyncio.create_task(self._handle_command_request(msg))
|
4454
|
+
self._cmds_by_seq[msg.seq] = _RemoteCommandHandler._Command(
|
4455
|
+
req=msg,
|
4456
|
+
fut=fut,
|
4457
|
+
)
|
4458
|
+
|
4459
|
+
else:
|
4460
|
+
raise TypeError(msg)
|
4461
|
+
|
4462
|
+
async def _handle_command_request(self, req: _RemoteProtocol.CommandRequest) -> None:
|
4463
|
+
res = await self._executor.try_execute(
|
4464
|
+
req.cmd,
|
4465
|
+
log=log,
|
4466
|
+
omit_exc_object=True,
|
4467
|
+
)
|
4468
|
+
|
4469
|
+
await _RemoteProtocol.CommandResponse(
|
4470
|
+
seq=req.seq,
|
4471
|
+
res=CommandOutputOrExceptionData(
|
4472
|
+
output=res.output,
|
4473
|
+
exception=res.exception,
|
4474
|
+
),
|
4475
|
+
).send(self._chan)
|
4476
|
+
|
4477
|
+
self._cmds_by_seq.pop(req.seq) # noqa
|
4478
|
+
|
4479
|
+
|
4480
|
+
##
|
4481
|
+
|
4482
|
+
|
4483
|
+
@dc.dataclass()
|
4484
|
+
class RemoteCommandError(Exception):
|
4485
|
+
e: CommandException
|
4486
|
+
|
4487
|
+
|
4488
|
+
class RemoteCommandExecutor(CommandExecutor):
|
4489
|
+
def __init__(self, chan: RemoteChannel) -> None:
|
4490
|
+
super().__init__()
|
4491
|
+
|
4492
|
+
self._chan = chan
|
4493
|
+
|
4494
|
+
self._cmd_seq = itertools.count()
|
4495
|
+
self._queue: asyncio.Queue = asyncio.Queue() # asyncio.Queue[RemoteCommandExecutor._Request]
|
4496
|
+
self._stop = asyncio.Event()
|
4497
|
+
self._loop_task: ta.Optional[asyncio.Task] = None
|
4498
|
+
self._reqs_by_seq: ta.Dict[int, RemoteCommandExecutor._Request] = {}
|
4499
|
+
|
4500
|
+
#
|
4501
|
+
|
4502
|
+
async def start(self) -> None:
|
4503
|
+
check_none(self._loop_task)
|
4504
|
+
check_state(not self._stop.is_set())
|
4505
|
+
self._loop_task = asyncio.create_task(self._loop())
|
4506
|
+
|
4507
|
+
async def aclose(self) -> None:
|
4508
|
+
self._stop.set()
|
4509
|
+
if self._loop_task is not None:
|
4510
|
+
await self._loop_task
|
4511
|
+
|
4512
|
+
#
|
4513
|
+
|
4514
|
+
@dc.dataclass(frozen=True)
|
4515
|
+
class _Request:
|
4516
|
+
seq: int
|
4517
|
+
cmd: Command
|
4518
|
+
fut: asyncio.Future
|
4519
|
+
|
4520
|
+
async def _loop(self) -> None:
|
4521
|
+
log.debug('RemoteCommandExecutor loop start: %r', self)
|
4522
|
+
|
4523
|
+
stop_task = asyncio.create_task(self._stop.wait())
|
4524
|
+
queue_task: ta.Optional[asyncio.Task] = None
|
4525
|
+
recv_task: ta.Optional[asyncio.Task] = None
|
4526
|
+
|
4527
|
+
while not self._stop.is_set():
|
4528
|
+
if queue_task is None:
|
4529
|
+
queue_task = asyncio.create_task(self._queue.get())
|
4530
|
+
if recv_task is None:
|
4531
|
+
recv_task = asyncio.create_task(_RemoteProtocol.Message.recv(self._chan))
|
4532
|
+
|
4533
|
+
done, pending = await asyncio.wait([
|
4534
|
+
stop_task,
|
4535
|
+
queue_task,
|
4536
|
+
recv_task,
|
4537
|
+
], return_when=asyncio.FIRST_COMPLETED)
|
4538
|
+
|
4539
|
+
if queue_task in done:
|
4540
|
+
req = check_isinstance(queue_task.result(), RemoteCommandExecutor._Request)
|
4541
|
+
queue_task = None
|
4542
|
+
await self._handle_request(req)
|
4543
|
+
|
4544
|
+
if recv_task in done:
|
4545
|
+
msg: ta.Optional[_RemoteProtocol.Message] = check_isinstance(
|
4546
|
+
recv_task.result(),
|
4547
|
+
(_RemoteProtocol.Message, type(None)),
|
4548
|
+
)
|
4549
|
+
recv_task = None
|
4550
|
+
|
4551
|
+
if msg is None:
|
4552
|
+
log.debug('RemoteCommandExecutor got eof: %r', self)
|
4553
|
+
break
|
4554
|
+
|
4555
|
+
await self._handle_message(msg)
|
4556
|
+
|
4557
|
+
log.debug('RemoteCommandExecutor loop stopping: %r', self)
|
4558
|
+
|
4559
|
+
for task in [
|
4560
|
+
stop_task,
|
4561
|
+
queue_task,
|
4562
|
+
recv_task,
|
4563
|
+
]:
|
4564
|
+
if task is not None and not task.done():
|
4565
|
+
task.cancel()
|
4566
|
+
|
4567
|
+
for req in self._reqs_by_seq.values():
|
4568
|
+
req.fut.cancel()
|
4569
|
+
|
4570
|
+
log.debug('RemoteCommandExecutor loop exited: %r', self)
|
4571
|
+
|
4572
|
+
async def _handle_request(self, req: _Request) -> None:
|
4573
|
+
self._reqs_by_seq[req.seq] = req
|
4574
|
+
await _RemoteProtocol.CommandRequest(
|
4575
|
+
seq=req.seq,
|
4576
|
+
cmd=req.cmd,
|
4577
|
+
).send(self._chan)
|
4578
|
+
|
4579
|
+
async def _handle_message(self, msg: _RemoteProtocol.Message) -> None:
|
4580
|
+
if isinstance(msg, _RemoteProtocol.PingRequest):
|
4581
|
+
log.debug('Ping: %r', msg)
|
4582
|
+
await _RemoteProtocol.PingResponse(
|
4583
|
+
time=msg.time,
|
4584
|
+
).send(self._chan)
|
4585
|
+
|
4586
|
+
elif isinstance(msg, _RemoteProtocol.LogResponse):
|
4587
|
+
log.info(msg.s)
|
4588
|
+
|
4589
|
+
elif isinstance(msg, _RemoteProtocol.CommandResponse):
|
4590
|
+
req = self._reqs_by_seq.pop(msg.seq)
|
4591
|
+
req.fut.set_result(msg.res)
|
4592
|
+
|
4593
|
+
else:
|
4594
|
+
raise TypeError(msg)
|
4595
|
+
|
4596
|
+
#
|
4597
|
+
|
4598
|
+
async def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
|
4599
|
+
req = RemoteCommandExecutor._Request(
|
4600
|
+
seq=next(self._cmd_seq),
|
4601
|
+
cmd=cmd,
|
4602
|
+
fut=asyncio.Future(),
|
4603
|
+
)
|
4604
|
+
await self._queue.put(req)
|
4605
|
+
return await req.fut
|
4606
|
+
|
4607
|
+
# @ta.override
|
4608
|
+
async def execute(self, cmd: Command) -> Command.Output:
|
4609
|
+
r = await self._remote_execute(cmd)
|
4610
|
+
if (e := r.exception) is not None:
|
4611
|
+
raise RemoteCommandError(e)
|
4612
|
+
else:
|
4613
|
+
return check_not_none(r.output)
|
4614
|
+
|
4615
|
+
# @ta.override
|
4616
|
+
async def try_execute(
|
4617
|
+
self,
|
4618
|
+
cmd: Command,
|
4619
|
+
*,
|
4620
|
+
log: ta.Optional[logging.Logger] = None,
|
4621
|
+
omit_exc_object: bool = False,
|
4622
|
+
) -> CommandOutputOrException:
|
4623
|
+
try:
|
4624
|
+
r = await self._remote_execute(cmd)
|
4625
|
+
|
4626
|
+
except Exception as e: # noqa
|
4627
|
+
if log is not None:
|
4628
|
+
log.exception('Exception executing remote command: %r', type(cmd))
|
4629
|
+
|
4630
|
+
return CommandOutputOrExceptionData(exception=CommandException.of(
|
4631
|
+
e,
|
4632
|
+
omit_exc_object=omit_exc_object,
|
4633
|
+
cmd=cmd,
|
4634
|
+
))
|
4635
|
+
|
4636
|
+
else:
|
4637
|
+
return r
|
4638
|
+
|
4639
|
+
|
4640
|
+
########################################
|
4641
|
+
# ../../../omlish/lite/asyncio/subprocesses.py
|
4642
|
+
|
4643
|
+
|
4644
|
+
##
|
4645
|
+
|
4646
|
+
|
4647
|
+
@contextlib.asynccontextmanager
|
4648
|
+
async def asyncio_subprocess_popen(
|
4649
|
+
*cmd: str,
|
4650
|
+
shell: bool = False,
|
4651
|
+
timeout: ta.Optional[float] = None,
|
4652
|
+
**kwargs: ta.Any,
|
4653
|
+
) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
|
4654
|
+
fac: ta.Any
|
4655
|
+
if shell:
|
4656
|
+
fac = functools.partial(
|
4657
|
+
asyncio.create_subprocess_shell,
|
4658
|
+
check_single(cmd),
|
4659
|
+
)
|
4660
|
+
else:
|
4661
|
+
fac = functools.partial(
|
4662
|
+
asyncio.create_subprocess_exec,
|
4663
|
+
*cmd,
|
4664
|
+
)
|
4665
|
+
|
4666
|
+
with subprocess_common_context(
|
4667
|
+
*cmd,
|
4668
|
+
shell=shell,
|
4669
|
+
timeout=timeout,
|
4670
|
+
**kwargs,
|
4671
|
+
):
|
4672
|
+
proc: asyncio.subprocess.Process
|
4673
|
+
proc = await fac(**kwargs)
|
4674
|
+
try:
|
4675
|
+
yield proc
|
4676
|
+
|
4677
|
+
finally:
|
4678
|
+
await asyncio_maybe_timeout(proc.wait(), timeout)
|
4679
|
+
|
4680
|
+
|
4681
|
+
##
|
4682
|
+
|
4683
|
+
|
4684
|
+
class AsyncioProcessCommunicator:
|
4685
|
+
def __init__(
|
4686
|
+
self,
|
4687
|
+
proc: asyncio.subprocess.Process,
|
4688
|
+
loop: ta.Optional[ta.Any] = None,
|
4689
|
+
) -> None:
|
4690
|
+
super().__init__()
|
4691
|
+
|
4692
|
+
if loop is None:
|
4693
|
+
loop = asyncio.get_running_loop()
|
4694
|
+
|
4695
|
+
self._proc = proc
|
4696
|
+
self._loop = loop
|
4697
|
+
|
4698
|
+
self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check_isinstance(
|
4699
|
+
proc._transport, # type: ignore # noqa
|
4700
|
+
asyncio.base_subprocess.BaseSubprocessTransport,
|
4701
|
+
)
|
4702
|
+
|
4703
|
+
@property
|
4704
|
+
def _debug(self) -> bool:
|
4705
|
+
return self._loop.get_debug()
|
4706
|
+
|
4707
|
+
async def _feed_stdin(self, input: bytes) -> None: # noqa
|
4708
|
+
stdin = check_not_none(self._proc.stdin)
|
4709
|
+
try:
|
4710
|
+
if input is not None:
|
4711
|
+
stdin.write(input)
|
4712
|
+
if self._debug:
|
4713
|
+
log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
|
4714
|
+
|
4715
|
+
await stdin.drain()
|
4716
|
+
|
4717
|
+
except (BrokenPipeError, ConnectionResetError) as exc:
|
4718
|
+
# communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
|
4719
|
+
# exceptions.
|
4720
|
+
if self._debug:
|
4721
|
+
log.debug('%r communicate: stdin got %r', self, exc)
|
4722
|
+
|
4723
|
+
if self._debug:
|
4724
|
+
log.debug('%r communicate: close stdin', self)
|
4725
|
+
|
4726
|
+
stdin.close()
|
4727
|
+
|
4728
|
+
async def _noop(self) -> None:
|
4729
|
+
return None
|
4730
|
+
|
4731
|
+
async def _read_stream(self, fd: int) -> bytes:
|
4732
|
+
transport: ta.Any = check_not_none(self._transport.get_pipe_transport(fd))
|
4733
|
+
|
4734
|
+
if fd == 2:
|
4735
|
+
stream = check_not_none(self._proc.stderr)
|
4736
|
+
else:
|
4737
|
+
check_equal(fd, 1)
|
4738
|
+
stream = check_not_none(self._proc.stdout)
|
4739
|
+
|
4740
|
+
if self._debug:
|
4741
|
+
name = 'stdout' if fd == 1 else 'stderr'
|
4742
|
+
log.debug('%r communicate: read %s', self, name)
|
4743
|
+
|
4744
|
+
output = await stream.read()
|
4745
|
+
|
4746
|
+
if self._debug:
|
4747
|
+
name = 'stdout' if fd == 1 else 'stderr'
|
4748
|
+
log.debug('%r communicate: close %s', self, name)
|
4749
|
+
|
4750
|
+
transport.close()
|
4751
|
+
|
4752
|
+
return output
|
4753
|
+
|
4754
|
+
class Communication(ta.NamedTuple):
|
4755
|
+
stdout: ta.Optional[bytes]
|
4756
|
+
stderr: ta.Optional[bytes]
|
4757
|
+
|
4758
|
+
async def _communicate(
|
4759
|
+
self,
|
4760
|
+
input: ta.Any = None, # noqa
|
4761
|
+
) -> Communication:
|
4762
|
+
stdin_fut: ta.Any
|
4763
|
+
if self._proc.stdin is not None:
|
4764
|
+
stdin_fut = self._feed_stdin(input)
|
4765
|
+
else:
|
4766
|
+
stdin_fut = self._noop()
|
4767
|
+
|
4768
|
+
stdout_fut: ta.Any
|
4769
|
+
if self._proc.stdout is not None:
|
4770
|
+
stdout_fut = self._read_stream(1)
|
4771
|
+
else:
|
4772
|
+
stdout_fut = self._noop()
|
4773
|
+
|
4774
|
+
stderr_fut: ta.Any
|
4775
|
+
if self._proc.stderr is not None:
|
4776
|
+
stderr_fut = self._read_stream(2)
|
4777
|
+
else:
|
4778
|
+
stderr_fut = self._noop()
|
4779
|
+
|
4780
|
+
stdin_res, stdout_res, stderr_res = await asyncio.gather(stdin_fut, stdout_fut, stderr_fut)
|
4781
|
+
|
4782
|
+
await self._proc.wait()
|
4783
|
+
|
4784
|
+
return AsyncioProcessCommunicator.Communication(stdout_res, stderr_res)
|
4785
|
+
|
4786
|
+
async def communicate(
|
4787
|
+
self,
|
4788
|
+
input: ta.Any = None, # noqa
|
4789
|
+
timeout: ta.Optional[float] = None,
|
4790
|
+
) -> Communication:
|
4791
|
+
return await asyncio_maybe_timeout(self._communicate(input), timeout)
|
4792
|
+
|
4793
|
+
|
4794
|
+
async def asyncio_subprocess_communicate(
|
4795
|
+
proc: asyncio.subprocess.Process,
|
4796
|
+
input: ta.Any = None, # noqa
|
4797
|
+
timeout: ta.Optional[float] = None,
|
4798
|
+
) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
|
4799
|
+
return await AsyncioProcessCommunicator(proc).communicate(input, timeout) # noqa
|
4800
|
+
|
4801
|
+
|
4802
|
+
##
|
4803
|
+
|
4804
|
+
|
4805
|
+
async def _asyncio_subprocess_check_run(
|
4806
|
+
*args: str,
|
4807
|
+
input: ta.Any = None, # noqa
|
4808
|
+
timeout: ta.Optional[float] = None,
|
4809
|
+
**kwargs: ta.Any,
|
4810
|
+
) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
|
4811
|
+
args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
|
4812
|
+
|
4813
|
+
proc: asyncio.subprocess.Process
|
4814
|
+
async with asyncio_subprocess_popen(*args, **kwargs) as proc:
|
4815
|
+
stdout, stderr = await asyncio_subprocess_communicate(proc, input, timeout)
|
4816
|
+
|
4817
|
+
if proc.returncode:
|
4818
|
+
raise subprocess.CalledProcessError(
|
4819
|
+
proc.returncode,
|
4820
|
+
args,
|
4821
|
+
output=stdout,
|
4822
|
+
stderr=stderr,
|
4823
|
+
)
|
4824
|
+
|
4825
|
+
return stdout, stderr
|
4826
|
+
|
4827
|
+
|
4828
|
+
async def asyncio_subprocess_check_call(
|
4829
|
+
*args: str,
|
4830
|
+
stdout: ta.Any = sys.stderr,
|
4831
|
+
input: ta.Any = None, # noqa
|
4832
|
+
timeout: ta.Optional[float] = None,
|
4833
|
+
**kwargs: ta.Any,
|
4834
|
+
) -> None:
|
4835
|
+
_, _ = await _asyncio_subprocess_check_run(
|
4836
|
+
*args,
|
4837
|
+
stdout=stdout,
|
4838
|
+
input=input,
|
4839
|
+
timeout=timeout,
|
4840
|
+
**kwargs,
|
4841
|
+
)
|
4842
|
+
|
4843
|
+
|
4844
|
+
async def asyncio_subprocess_check_output(
|
4845
|
+
*args: str,
|
4846
|
+
input: ta.Any = None, # noqa
|
4847
|
+
timeout: ta.Optional[float] = None,
|
4848
|
+
**kwargs: ta.Any,
|
4849
|
+
) -> bytes:
|
4850
|
+
stdout, stderr = await _asyncio_subprocess_check_run(
|
4851
|
+
*args,
|
4852
|
+
stdout=asyncio.subprocess.PIPE,
|
4853
|
+
input=input,
|
4854
|
+
timeout=timeout,
|
4855
|
+
**kwargs,
|
4856
|
+
)
|
4857
|
+
|
4858
|
+
return check_not_none(stdout)
|
4859
|
+
|
4860
|
+
|
4861
|
+
async def asyncio_subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
|
4862
|
+
return (await asyncio_subprocess_check_output(*args, **kwargs)).decode().strip()
|
4863
|
+
|
4864
|
+
|
4865
|
+
##
|
4866
|
+
|
4867
|
+
|
4868
|
+
async def _asyncio_subprocess_try_run(
|
4869
|
+
fn: ta.Callable[..., ta.Awaitable[T]],
|
4870
|
+
*args: ta.Any,
|
4871
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4872
|
+
**kwargs: ta.Any,
|
4873
|
+
) -> ta.Union[T, Exception]:
|
4874
|
+
try:
|
4875
|
+
return await fn(*args, **kwargs)
|
4876
|
+
except try_exceptions as e: # noqa
|
4877
|
+
if log.isEnabledFor(logging.DEBUG):
|
4878
|
+
log.exception('command failed')
|
4879
|
+
return e
|
4880
|
+
|
4881
|
+
|
4882
|
+
async def asyncio_subprocess_try_call(
|
4883
|
+
*args: str,
|
4884
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4885
|
+
**kwargs: ta.Any,
|
4886
|
+
) -> bool:
|
4887
|
+
if isinstance(await _asyncio_subprocess_try_run(
|
4888
|
+
asyncio_subprocess_check_call,
|
4889
|
+
*args,
|
4890
|
+
try_exceptions=try_exceptions,
|
4891
|
+
**kwargs,
|
4892
|
+
), Exception):
|
4893
|
+
return False
|
4894
|
+
else:
|
4895
|
+
return True
|
4896
|
+
|
4897
|
+
|
4898
|
+
async def asyncio_subprocess_try_output(
|
4899
|
+
*args: str,
|
4900
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4901
|
+
**kwargs: ta.Any,
|
4902
|
+
) -> ta.Optional[bytes]:
|
4903
|
+
if isinstance(ret := await _asyncio_subprocess_try_run(
|
4904
|
+
asyncio_subprocess_check_output,
|
4905
|
+
*args,
|
4906
|
+
try_exceptions=try_exceptions,
|
4907
|
+
**kwargs,
|
4908
|
+
), Exception):
|
4909
|
+
return None
|
4910
|
+
else:
|
4911
|
+
return ret
|
4912
|
+
|
4913
|
+
|
4914
|
+
async def asyncio_subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|
4915
|
+
out = await asyncio_subprocess_try_output(*args, **kwargs)
|
4916
|
+
return out.decode().strip() if out is not None else None
|
4917
|
+
|
4918
|
+
|
4919
|
+
########################################
|
4920
|
+
# ../../../omdev/interp/inspect.py
|
4921
|
+
|
4922
|
+
|
4923
|
+
@dc.dataclass(frozen=True)
|
4924
|
+
class InterpInspection:
|
4925
|
+
exe: str
|
4926
|
+
version: Version
|
4927
|
+
|
4928
|
+
version_str: str
|
4929
|
+
config_vars: ta.Mapping[str, str]
|
4930
|
+
prefix: str
|
4931
|
+
base_prefix: str
|
4932
|
+
|
4933
|
+
@property
|
4934
|
+
def opts(self) -> InterpOpts:
|
4935
|
+
return InterpOpts(
|
4936
|
+
threaded=bool(self.config_vars.get('Py_GIL_DISABLED')),
|
4937
|
+
debug=bool(self.config_vars.get('Py_DEBUG')),
|
4938
|
+
)
|
4939
|
+
|
4940
|
+
@property
|
4941
|
+
def iv(self) -> InterpVersion:
|
4942
|
+
return InterpVersion(
|
4943
|
+
version=self.version,
|
4944
|
+
opts=self.opts,
|
4945
|
+
)
|
4946
|
+
|
4947
|
+
@property
|
4948
|
+
def is_venv(self) -> bool:
|
4949
|
+
return self.prefix != self.base_prefix
|
4950
|
+
|
4951
|
+
|
4952
|
+
class InterpInspector:
|
4953
|
+
def __init__(self) -> None:
|
4954
|
+
super().__init__()
|
4955
|
+
|
4956
|
+
self._cache: ta.Dict[str, ta.Optional[InterpInspection]] = {}
|
4957
|
+
|
4958
|
+
_RAW_INSPECTION_CODE = """
|
4959
|
+
__import__('json').dumps(dict(
|
4960
|
+
version_str=__import__('sys').version,
|
4961
|
+
prefix=__import__('sys').prefix,
|
4962
|
+
base_prefix=__import__('sys').base_prefix,
|
4963
|
+
config_vars=__import__('sysconfig').get_config_vars(),
|
4964
|
+
))"""
|
4965
|
+
|
4966
|
+
_INSPECTION_CODE = ''.join(l.strip() for l in _RAW_INSPECTION_CODE.splitlines())
|
4967
|
+
|
4968
|
+
@staticmethod
|
4969
|
+
def _build_inspection(
|
4970
|
+
exe: str,
|
4971
|
+
output: str,
|
4972
|
+
) -> InterpInspection:
|
4973
|
+
dct = json.loads(output)
|
4974
|
+
|
4975
|
+
version = Version(dct['version_str'].split()[0])
|
4976
|
+
|
4977
|
+
return InterpInspection(
|
4978
|
+
exe=exe,
|
4979
|
+
version=version,
|
4980
|
+
**{k: dct[k] for k in (
|
4981
|
+
'version_str',
|
4982
|
+
'prefix',
|
4983
|
+
'base_prefix',
|
4984
|
+
'config_vars',
|
4985
|
+
)},
|
4986
|
+
)
|
4987
|
+
|
4988
|
+
@classmethod
|
4989
|
+
def running(cls) -> 'InterpInspection':
|
4990
|
+
return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
|
4991
|
+
|
4992
|
+
async def _inspect(self, exe: str) -> InterpInspection:
|
4993
|
+
output = await asyncio_subprocess_check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
|
4994
|
+
return self._build_inspection(exe, output.decode())
|
4995
|
+
|
4996
|
+
async def inspect(self, exe: str) -> ta.Optional[InterpInspection]:
|
4997
|
+
try:
|
4998
|
+
return self._cache[exe]
|
4999
|
+
except KeyError:
|
5000
|
+
ret: ta.Optional[InterpInspection]
|
5001
|
+
try:
|
5002
|
+
ret = await self._inspect(exe)
|
5003
|
+
except Exception as e: # noqa
|
5004
|
+
if log.isEnabledFor(logging.DEBUG):
|
5005
|
+
log.exception('Failed to inspect interp: %s', exe)
|
5006
|
+
ret = None
|
5007
|
+
self._cache[exe] = ret
|
5008
|
+
return ret
|
5009
|
+
|
5010
|
+
|
5011
|
+
INTERP_INSPECTOR = InterpInspector()
|
5012
|
+
|
5013
|
+
|
5014
|
+
########################################
|
5015
|
+
# ../commands/subprocess.py
|
5016
|
+
|
5017
|
+
|
5018
|
+
##
|
5019
|
+
|
5020
|
+
|
5021
|
+
@dc.dataclass(frozen=True)
|
5022
|
+
class SubprocessCommand(Command['SubprocessCommand.Output']):
|
5023
|
+
cmd: ta.Sequence[str]
|
5024
|
+
|
5025
|
+
shell: bool = False
|
5026
|
+
cwd: ta.Optional[str] = None
|
5027
|
+
env: ta.Optional[ta.Mapping[str, str]] = None
|
5028
|
+
|
5029
|
+
stdout: str = 'pipe' # SubprocessChannelOption
|
5030
|
+
stderr: str = 'pipe' # SubprocessChannelOption
|
5031
|
+
|
5032
|
+
input: ta.Optional[bytes] = None
|
5033
|
+
timeout: ta.Optional[float] = None
|
4208
5034
|
|
4209
5035
|
def __post_init__(self) -> None:
|
4210
5036
|
check_not_isinstance(self.cmd, str)
|
4211
5037
|
|
4212
|
-
@dc.dataclass(frozen=True)
|
4213
|
-
class Output(Command.Output):
|
4214
|
-
rc: int
|
4215
|
-
pid: int
|
5038
|
+
@dc.dataclass(frozen=True)
|
5039
|
+
class Output(Command.Output):
|
5040
|
+
rc: int
|
5041
|
+
pid: int
|
5042
|
+
|
5043
|
+
elapsed_s: float
|
5044
|
+
|
5045
|
+
stdout: ta.Optional[bytes] = None
|
5046
|
+
stderr: ta.Optional[bytes] = None
|
5047
|
+
|
5048
|
+
|
5049
|
+
##
|
5050
|
+
|
5051
|
+
|
5052
|
+
class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
|
5053
|
+
async def execute(self, cmd: SubprocessCommand) -> SubprocessCommand.Output:
|
5054
|
+
proc: asyncio.subprocess.Process
|
5055
|
+
async with asyncio_subprocess_popen(
|
5056
|
+
*subprocess_maybe_shell_wrap_exec(*cmd.cmd),
|
5057
|
+
|
5058
|
+
shell=cmd.shell,
|
5059
|
+
cwd=cmd.cwd,
|
5060
|
+
env={**os.environ, **(cmd.env or {})},
|
5061
|
+
|
5062
|
+
stdin=subprocess.PIPE if cmd.input is not None else None,
|
5063
|
+
stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, cmd.stdout)],
|
5064
|
+
stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, cmd.stderr)],
|
5065
|
+
|
5066
|
+
timeout=cmd.timeout,
|
5067
|
+
) as proc:
|
5068
|
+
start_time = time.time()
|
5069
|
+
stdout, stderr = await asyncio_subprocess_communicate(
|
5070
|
+
proc,
|
5071
|
+
input=cmd.input,
|
5072
|
+
timeout=cmd.timeout,
|
5073
|
+
)
|
5074
|
+
end_time = time.time()
|
5075
|
+
|
5076
|
+
return SubprocessCommand.Output(
|
5077
|
+
rc=check_not_none(proc.returncode),
|
5078
|
+
pid=proc.pid,
|
5079
|
+
|
5080
|
+
elapsed_s=end_time - start_time,
|
5081
|
+
|
5082
|
+
stdout=stdout, # noqa
|
5083
|
+
stderr=stderr, # noqa
|
5084
|
+
)
|
5085
|
+
|
5086
|
+
|
5087
|
+
########################################
|
5088
|
+
# ../remote/_main.py
|
5089
|
+
|
5090
|
+
|
5091
|
+
##
|
5092
|
+
|
5093
|
+
|
5094
|
+
class _RemoteExecutionLogHandler(logging.Handler):
|
5095
|
+
def __init__(self, fn: ta.Callable[[str], None]) -> None:
|
5096
|
+
super().__init__()
|
5097
|
+
self._fn = fn
|
5098
|
+
|
5099
|
+
def emit(self, record):
|
5100
|
+
msg = self.format(record)
|
5101
|
+
self._fn(msg)
|
5102
|
+
|
5103
|
+
|
5104
|
+
##
|
5105
|
+
|
5106
|
+
|
5107
|
+
class _RemoteExecutionMain:
|
5108
|
+
def __init__(
|
5109
|
+
self,
|
5110
|
+
chan: RemoteChannel,
|
5111
|
+
) -> None:
|
5112
|
+
super().__init__()
|
5113
|
+
|
5114
|
+
self._chan = chan
|
5115
|
+
|
5116
|
+
self.__bootstrap: ta.Optional[MainBootstrap] = None
|
5117
|
+
self.__injector: ta.Optional[Injector] = None
|
5118
|
+
|
5119
|
+
@property
|
5120
|
+
def _bootstrap(self) -> MainBootstrap:
|
5121
|
+
return check_not_none(self.__bootstrap)
|
5122
|
+
|
5123
|
+
@property
|
5124
|
+
def _injector(self) -> Injector:
|
5125
|
+
return check_not_none(self.__injector)
|
5126
|
+
|
5127
|
+
#
|
5128
|
+
|
5129
|
+
def _timebomb_main(
|
5130
|
+
self,
|
5131
|
+
delay_s: float,
|
5132
|
+
*,
|
5133
|
+
sig: int = signal.SIGINT,
|
5134
|
+
code: int = 1,
|
5135
|
+
) -> None:
|
5136
|
+
time.sleep(delay_s)
|
5137
|
+
|
5138
|
+
if (pgid := os.getpgid(0)) == os.getpid():
|
5139
|
+
os.killpg(pgid, sig)
|
5140
|
+
|
5141
|
+
os._exit(code) # noqa
|
5142
|
+
|
5143
|
+
@cached_nullary
|
5144
|
+
def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
|
5145
|
+
if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
|
5146
|
+
return None
|
5147
|
+
|
5148
|
+
thr = threading.Thread(
|
5149
|
+
target=functools.partial(self._timebomb_main, tbd),
|
5150
|
+
name=f'{self.__class__.__name__}.timebomb',
|
5151
|
+
daemon=True,
|
5152
|
+
)
|
5153
|
+
|
5154
|
+
thr.start()
|
4216
5155
|
|
4217
|
-
|
5156
|
+
log.debug('Started timebomb thread: %r', thr)
|
4218
5157
|
|
4219
|
-
|
4220
|
-
stderr: ta.Optional[bytes] = None
|
5158
|
+
return thr
|
4221
5159
|
|
5160
|
+
#
|
4222
5161
|
|
4223
|
-
|
5162
|
+
@cached_nullary
|
5163
|
+
def _log_handler(self) -> _RemoteLogHandler:
|
5164
|
+
return _RemoteLogHandler(self._chan)
|
4224
5165
|
|
5166
|
+
#
|
4225
5167
|
|
4226
|
-
|
4227
|
-
|
4228
|
-
|
4229
|
-
subprocess_maybe_shell_wrap_exec(*inp.cmd),
|
5168
|
+
async def _setup(self) -> None:
|
5169
|
+
check_none(self.__bootstrap)
|
5170
|
+
check_none(self.__injector)
|
4230
5171
|
|
4231
|
-
|
4232
|
-
cwd=inp.cwd,
|
4233
|
-
env={**os.environ, **(inp.env or {})},
|
5172
|
+
# Bootstrap
|
4234
5173
|
|
4235
|
-
|
4236
|
-
stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stdout)],
|
4237
|
-
stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stderr)],
|
4238
|
-
) as proc:
|
4239
|
-
start_time = time.time()
|
4240
|
-
stdout, stderr = proc.communicate(
|
4241
|
-
input=inp.input,
|
4242
|
-
timeout=inp.timeout,
|
4243
|
-
)
|
4244
|
-
end_time = time.time()
|
5174
|
+
self.__bootstrap = check_not_none(await self._chan.recv_obj(MainBootstrap))
|
4245
5175
|
|
4246
|
-
|
4247
|
-
|
4248
|
-
pid=proc.pid,
|
5176
|
+
if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
|
5177
|
+
pycharm_debug_connect(prd)
|
4249
5178
|
|
4250
|
-
|
5179
|
+
self.__injector = main_bootstrap(self._bootstrap)
|
4251
5180
|
|
4252
|
-
|
4253
|
-
|
5181
|
+
self._chan.set_marshaler(self._injector[ObjMarshalerManager])
|
5182
|
+
|
5183
|
+
# Post-bootstrap
|
5184
|
+
|
5185
|
+
if self._bootstrap.remote_config.set_pgid:
|
5186
|
+
if os.getpgid(0) != os.getpid():
|
5187
|
+
log.debug('Setting pgid')
|
5188
|
+
os.setpgid(0, 0)
|
5189
|
+
|
5190
|
+
if (ds := self._bootstrap.remote_config.deathsig) is not None:
|
5191
|
+
log.debug('Setting deathsig: %s', ds)
|
5192
|
+
set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
|
5193
|
+
|
5194
|
+
self._timebomb_thread()
|
5195
|
+
|
5196
|
+
if self._bootstrap.remote_config.forward_logging:
|
5197
|
+
log.debug('Installing log forwarder')
|
5198
|
+
logging.root.addHandler(self._log_handler())
|
5199
|
+
|
5200
|
+
#
|
5201
|
+
|
5202
|
+
async def run(self) -> None:
|
5203
|
+
await self._setup()
|
5204
|
+
|
5205
|
+
executor = self._injector[LocalCommandExecutor]
|
5206
|
+
|
5207
|
+
handler = _RemoteCommandHandler(self._chan, executor)
|
5208
|
+
|
5209
|
+
await handler.run()
|
5210
|
+
|
5211
|
+
|
5212
|
+
def _remote_execution_main() -> None:
|
5213
|
+
rt = pyremote_bootstrap_finalize() # noqa
|
5214
|
+
|
5215
|
+
async def inner() -> None:
|
5216
|
+
input = await asyncio_open_stream_reader(rt.input) # noqa
|
5217
|
+
output = await asyncio_open_stream_writer(rt.output)
|
5218
|
+
|
5219
|
+
chan = RemoteChannelImpl(
|
5220
|
+
input,
|
5221
|
+
output,
|
4254
5222
|
)
|
4255
5223
|
|
5224
|
+
await _RemoteExecutionMain(chan).run()
|
5225
|
+
|
5226
|
+
asyncio.run(inner())
|
5227
|
+
|
4256
5228
|
|
4257
5229
|
########################################
|
4258
5230
|
# ../remote/spawning.py
|
4259
5231
|
|
4260
5232
|
|
4261
|
-
|
5233
|
+
##
|
5234
|
+
|
5235
|
+
|
5236
|
+
class RemoteSpawning(abc.ABC):
|
4262
5237
|
@dc.dataclass(frozen=True)
|
4263
5238
|
class Target:
|
4264
5239
|
shell: ta.Optional[str] = None
|
@@ -4269,15 +5244,35 @@ class RemoteSpawning:
|
|
4269
5244
|
|
4270
5245
|
stderr: ta.Optional[str] = None # SubprocessChannelOption
|
4271
5246
|
|
4272
|
-
|
5247
|
+
@dc.dataclass(frozen=True)
|
5248
|
+
class Spawned:
|
5249
|
+
stdin: asyncio.StreamWriter
|
5250
|
+
stdout: asyncio.StreamReader
|
5251
|
+
stderr: ta.Optional[asyncio.StreamReader]
|
5252
|
+
|
5253
|
+
@abc.abstractmethod
|
5254
|
+
def spawn(
|
5255
|
+
self,
|
5256
|
+
tgt: Target,
|
5257
|
+
src: str,
|
5258
|
+
*,
|
5259
|
+
timeout: ta.Optional[float] = None,
|
5260
|
+
debug: bool = False,
|
5261
|
+
) -> ta.AsyncContextManager[Spawned]:
|
5262
|
+
raise NotImplementedError
|
4273
5263
|
|
4274
|
-
|
5264
|
+
|
5265
|
+
##
|
5266
|
+
|
5267
|
+
|
5268
|
+
class SubprocessRemoteSpawning(RemoteSpawning):
|
5269
|
+
class _PreparedCmd(ta.NamedTuple): # noqa
|
4275
5270
|
cmd: ta.Sequence[str]
|
4276
5271
|
shell: bool
|
4277
5272
|
|
4278
5273
|
def _prepare_cmd(
|
4279
5274
|
self,
|
4280
|
-
tgt: Target,
|
5275
|
+
tgt: RemoteSpawning.Target,
|
4281
5276
|
src: str,
|
4282
5277
|
) -> _PreparedCmd:
|
4283
5278
|
if tgt.shell is not None:
|
@@ -4285,44 +5280,38 @@ class RemoteSpawning:
|
|
4285
5280
|
if tgt.shell_quote:
|
4286
5281
|
sh_src = shlex.quote(sh_src)
|
4287
5282
|
sh_cmd = f'{tgt.shell} {sh_src}'
|
4288
|
-
return
|
4289
|
-
cmd=[sh_cmd],
|
4290
|
-
shell=True,
|
4291
|
-
)
|
5283
|
+
return SubprocessRemoteSpawning._PreparedCmd([sh_cmd], shell=True)
|
4292
5284
|
|
4293
5285
|
else:
|
4294
|
-
return
|
4295
|
-
cmd=[tgt.python, '-c', src],
|
4296
|
-
shell=False,
|
4297
|
-
)
|
5286
|
+
return SubprocessRemoteSpawning._PreparedCmd([tgt.python, '-c', src], shell=False)
|
4298
5287
|
|
4299
5288
|
#
|
4300
5289
|
|
4301
|
-
@
|
4302
|
-
|
4303
|
-
stdin: ta.IO
|
4304
|
-
stdout: ta.IO
|
4305
|
-
stderr: ta.Optional[ta.IO]
|
4306
|
-
|
4307
|
-
@contextlib.contextmanager
|
4308
|
-
def spawn(
|
5290
|
+
@contextlib.asynccontextmanager
|
5291
|
+
async def spawn(
|
4309
5292
|
self,
|
4310
|
-
tgt: Target,
|
5293
|
+
tgt: RemoteSpawning.Target,
|
4311
5294
|
src: str,
|
4312
5295
|
*,
|
4313
5296
|
timeout: ta.Optional[float] = None,
|
4314
|
-
|
5297
|
+
debug: bool = False,
|
5298
|
+
) -> ta.AsyncGenerator[RemoteSpawning.Spawned, None]:
|
4315
5299
|
pc = self._prepare_cmd(tgt, src)
|
4316
5300
|
|
4317
|
-
|
4318
|
-
|
4319
|
-
|
4320
|
-
|
4321
|
-
|
4322
|
-
|
4323
|
-
|
4324
|
-
|
4325
|
-
|
5301
|
+
cmd = pc.cmd
|
5302
|
+
if not debug:
|
5303
|
+
cmd = subprocess_maybe_shell_wrap_exec(*cmd)
|
5304
|
+
|
5305
|
+
async with asyncio_subprocess_popen(
|
5306
|
+
*cmd,
|
5307
|
+
shell=pc.shell,
|
5308
|
+
stdin=subprocess.PIPE,
|
5309
|
+
stdout=subprocess.PIPE,
|
5310
|
+
stderr=(
|
5311
|
+
SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, tgt.stderr)]
|
5312
|
+
if tgt.stderr is not None else None
|
5313
|
+
),
|
5314
|
+
timeout=timeout,
|
4326
5315
|
) as proc:
|
4327
5316
|
stdin = check_not_none(proc.stdin)
|
4328
5317
|
stdout = check_not_none(proc.stdout)
|
@@ -4340,8 +5329,6 @@ class RemoteSpawning:
|
|
4340
5329
|
except BrokenPipeError:
|
4341
5330
|
pass
|
4342
5331
|
|
4343
|
-
proc.wait(timeout)
|
4344
|
-
|
4345
5332
|
|
4346
5333
|
########################################
|
4347
5334
|
# ../../../omdev/interp/providers.py
|
@@ -4370,17 +5357,17 @@ class InterpProvider(abc.ABC):
|
|
4370
5357
|
setattr(cls, 'name', snake_case(cls.__name__[:-len(sfx)]))
|
4371
5358
|
|
4372
5359
|
@abc.abstractmethod
|
4373
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5360
|
+
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Awaitable[ta.Sequence[InterpVersion]]:
|
4374
5361
|
raise NotImplementedError
|
4375
5362
|
|
4376
5363
|
@abc.abstractmethod
|
4377
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5364
|
+
def get_installed_version(self, version: InterpVersion) -> ta.Awaitable[Interp]:
|
4378
5365
|
raise NotImplementedError
|
4379
5366
|
|
4380
|
-
def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5367
|
+
async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
4381
5368
|
return []
|
4382
5369
|
|
4383
|
-
def install_version(self, version: InterpVersion) -> Interp:
|
5370
|
+
async def install_version(self, version: InterpVersion) -> Interp:
|
4384
5371
|
raise TypeError
|
4385
5372
|
|
4386
5373
|
|
@@ -4392,10 +5379,10 @@ class RunningInterpProvider(InterpProvider):
|
|
4392
5379
|
def version(self) -> InterpVersion:
|
4393
5380
|
return InterpInspector.running().iv
|
4394
5381
|
|
4395
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5382
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
4396
5383
|
return [self.version()]
|
4397
5384
|
|
4398
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5385
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
4399
5386
|
if version != self.version():
|
4400
5387
|
raise KeyError(version)
|
4401
5388
|
return Interp(
|
@@ -4405,159 +5392,26 @@ class RunningInterpProvider(InterpProvider):
|
|
4405
5392
|
|
4406
5393
|
|
4407
5394
|
########################################
|
4408
|
-
# ../remote/
|
4409
|
-
|
4410
|
-
|
4411
|
-
##
|
4412
|
-
|
4413
|
-
|
4414
|
-
class _RemoteExecutionLogHandler(logging.Handler):
|
4415
|
-
def __init__(self, fn: ta.Callable[[str], None]) -> None:
|
4416
|
-
super().__init__()
|
4417
|
-
self._fn = fn
|
4418
|
-
|
4419
|
-
def emit(self, record):
|
4420
|
-
msg = self.format(record)
|
4421
|
-
self._fn(msg)
|
4422
|
-
|
4423
|
-
|
4424
|
-
@dc.dataclass(frozen=True)
|
4425
|
-
class _RemoteExecutionRequest:
|
4426
|
-
c: Command
|
4427
|
-
|
4428
|
-
|
4429
|
-
@dc.dataclass(frozen=True)
|
4430
|
-
class _RemoteExecutionLog:
|
4431
|
-
s: str
|
4432
|
-
|
4433
|
-
|
4434
|
-
@dc.dataclass(frozen=True)
|
4435
|
-
class _RemoteExecutionResponse:
|
4436
|
-
r: ta.Optional[CommandOutputOrExceptionData] = None
|
4437
|
-
l: ta.Optional[_RemoteExecutionLog] = None
|
4438
|
-
|
4439
|
-
|
4440
|
-
def _remote_execution_main() -> None:
|
4441
|
-
rt = pyremote_bootstrap_finalize() # noqa
|
4442
|
-
|
4443
|
-
chan = RemoteChannel(
|
4444
|
-
rt.input,
|
4445
|
-
rt.output,
|
4446
|
-
)
|
4447
|
-
|
4448
|
-
bs = check_not_none(chan.recv_obj(MainBootstrap))
|
4449
|
-
|
4450
|
-
if (prd := bs.remote_config.pycharm_remote_debug) is not None:
|
4451
|
-
pycharm_debug_connect(prd)
|
4452
|
-
|
4453
|
-
injector = main_bootstrap(bs)
|
4454
|
-
|
4455
|
-
chan.set_marshaler(injector[ObjMarshalerManager])
|
4456
|
-
|
4457
|
-
#
|
4458
|
-
|
4459
|
-
log_lock = threading.RLock()
|
4460
|
-
send_logs = False
|
4461
|
-
|
4462
|
-
def log_fn(s: str) -> None:
|
4463
|
-
with log_lock:
|
4464
|
-
if send_logs:
|
4465
|
-
chan.send_obj(_RemoteExecutionResponse(l=_RemoteExecutionLog(s)))
|
4466
|
-
|
4467
|
-
log_handler = _RemoteExecutionLogHandler(log_fn)
|
4468
|
-
logging.root.addHandler(log_handler)
|
4469
|
-
|
4470
|
-
#
|
4471
|
-
|
4472
|
-
ce = injector[LocalCommandExecutor]
|
4473
|
-
|
4474
|
-
while True:
|
4475
|
-
req = chan.recv_obj(_RemoteExecutionRequest)
|
4476
|
-
if req is None:
|
4477
|
-
break
|
4478
|
-
|
4479
|
-
with log_lock:
|
4480
|
-
send_logs = True
|
4481
|
-
|
4482
|
-
r = ce.try_execute(
|
4483
|
-
req.c,
|
4484
|
-
log=log,
|
4485
|
-
omit_exc_object=True,
|
4486
|
-
)
|
4487
|
-
|
4488
|
-
with log_lock:
|
4489
|
-
send_logs = False
|
4490
|
-
|
4491
|
-
chan.send_obj(_RemoteExecutionResponse(r=CommandOutputOrExceptionData(
|
4492
|
-
output=r.output,
|
4493
|
-
exception=r.exception,
|
4494
|
-
)))
|
5395
|
+
# ../remote/connection.py
|
4495
5396
|
|
4496
5397
|
|
4497
5398
|
##
|
4498
5399
|
|
4499
5400
|
|
4500
|
-
|
4501
|
-
|
4502
|
-
|
4503
|
-
|
4504
|
-
|
4505
|
-
class RemoteCommandExecutor(CommandExecutor):
|
4506
|
-
def __init__(self, chan: RemoteChannel) -> None:
|
4507
|
-
super().__init__()
|
4508
|
-
|
4509
|
-
self._chan = chan
|
4510
|
-
|
4511
|
-
def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
|
4512
|
-
self._chan.send_obj(_RemoteExecutionRequest(cmd))
|
4513
|
-
|
4514
|
-
while True:
|
4515
|
-
if (r := self._chan.recv_obj(_RemoteExecutionResponse)) is None:
|
4516
|
-
raise EOFError
|
4517
|
-
|
4518
|
-
if r.l is not None:
|
4519
|
-
log.info(r.l.s)
|
4520
|
-
|
4521
|
-
if r.r is not None:
|
4522
|
-
return r.r
|
4523
|
-
|
4524
|
-
# @ta.override
|
4525
|
-
def execute(self, cmd: Command) -> Command.Output:
|
4526
|
-
r = self._remote_execute(cmd)
|
4527
|
-
if (e := r.exception) is not None:
|
4528
|
-
raise RemoteCommandError(e)
|
4529
|
-
else:
|
4530
|
-
return check_not_none(r.output)
|
4531
|
-
|
4532
|
-
# @ta.override
|
4533
|
-
def try_execute(
|
5401
|
+
class RemoteExecutionConnector(abc.ABC):
|
5402
|
+
@abc.abstractmethod
|
5403
|
+
def connect(
|
4534
5404
|
self,
|
4535
|
-
|
4536
|
-
|
4537
|
-
|
4538
|
-
|
4539
|
-
) -> CommandOutputOrException:
|
4540
|
-
try:
|
4541
|
-
r = self._remote_execute(cmd)
|
4542
|
-
|
4543
|
-
except Exception as e: # noqa
|
4544
|
-
if log is not None:
|
4545
|
-
log.exception('Exception executing remote command: %r', type(cmd))
|
4546
|
-
|
4547
|
-
return CommandOutputOrExceptionData(exception=CommandException.of(
|
4548
|
-
e,
|
4549
|
-
omit_exc_object=omit_exc_object,
|
4550
|
-
cmd=cmd,
|
4551
|
-
))
|
4552
|
-
|
4553
|
-
else:
|
4554
|
-
return r
|
5405
|
+
tgt: RemoteSpawning.Target,
|
5406
|
+
bs: MainBootstrap,
|
5407
|
+
) -> ta.AsyncContextManager[RemoteCommandExecutor]:
|
5408
|
+
raise NotImplementedError
|
4555
5409
|
|
4556
5410
|
|
4557
5411
|
##
|
4558
5412
|
|
4559
5413
|
|
4560
|
-
class
|
5414
|
+
class PyremoteRemoteExecutionConnector(RemoteExecutionConnector):
|
4561
5415
|
def __init__(
|
4562
5416
|
self,
|
4563
5417
|
*,
|
@@ -4590,38 +5444,43 @@ class RemoteExecution:
|
|
4590
5444
|
|
4591
5445
|
#
|
4592
5446
|
|
4593
|
-
@contextlib.
|
4594
|
-
def connect(
|
5447
|
+
@contextlib.asynccontextmanager
|
5448
|
+
async def connect(
|
4595
5449
|
self,
|
4596
5450
|
tgt: RemoteSpawning.Target,
|
4597
5451
|
bs: MainBootstrap,
|
4598
|
-
) -> ta.
|
5452
|
+
) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
|
4599
5453
|
spawn_src = self._spawn_src()
|
4600
5454
|
remote_src = self._remote_src()
|
4601
5455
|
|
4602
|
-
with self._spawning.spawn(
|
5456
|
+
async with self._spawning.spawn(
|
4603
5457
|
tgt,
|
4604
5458
|
spawn_src,
|
5459
|
+
debug=bs.main_config.debug,
|
4605
5460
|
) as proc:
|
4606
|
-
res = PyremoteBootstrapDriver( # noqa
|
5461
|
+
res = await PyremoteBootstrapDriver( # noqa
|
4607
5462
|
remote_src,
|
4608
5463
|
PyremoteBootstrapOptions(
|
4609
5464
|
debug=bs.main_config.debug,
|
4610
5465
|
),
|
4611
|
-
).
|
5466
|
+
).async_run(
|
4612
5467
|
proc.stdout,
|
4613
5468
|
proc.stdin,
|
4614
5469
|
)
|
4615
5470
|
|
4616
|
-
chan =
|
5471
|
+
chan = RemoteChannelImpl(
|
4617
5472
|
proc.stdout,
|
4618
5473
|
proc.stdin,
|
4619
5474
|
msh=self._msh,
|
4620
5475
|
)
|
4621
5476
|
|
4622
|
-
chan.send_obj(bs)
|
5477
|
+
await chan.send_obj(bs)
|
5478
|
+
|
5479
|
+
rce: RemoteCommandExecutor
|
5480
|
+
async with contextlib.aclosing(RemoteCommandExecutor(chan)) as rce:
|
5481
|
+
await rce.start()
|
4623
5482
|
|
4624
|
-
|
5483
|
+
yield rce
|
4625
5484
|
|
4626
5485
|
|
4627
5486
|
########################################
|
@@ -4643,7 +5502,6 @@ TODO:
|
|
4643
5502
|
|
4644
5503
|
|
4645
5504
|
class Pyenv:
|
4646
|
-
|
4647
5505
|
def __init__(
|
4648
5506
|
self,
|
4649
5507
|
*,
|
@@ -4656,13 +5514,13 @@ class Pyenv:
|
|
4656
5514
|
|
4657
5515
|
self._root_kw = root
|
4658
5516
|
|
4659
|
-
@
|
4660
|
-
def root(self) -> ta.Optional[str]:
|
5517
|
+
@async_cached_nullary
|
5518
|
+
async def root(self) -> ta.Optional[str]:
|
4661
5519
|
if self._root_kw is not None:
|
4662
5520
|
return self._root_kw
|
4663
5521
|
|
4664
5522
|
if shutil.which('pyenv'):
|
4665
|
-
return
|
5523
|
+
return await asyncio_subprocess_check_output_str('pyenv', 'root')
|
4666
5524
|
|
4667
5525
|
d = os.path.expanduser('~/.pyenv')
|
4668
5526
|
if os.path.isdir(d) and os.path.isfile(os.path.join(d, 'bin', 'pyenv')):
|
@@ -4670,12 +5528,12 @@ class Pyenv:
|
|
4670
5528
|
|
4671
5529
|
return None
|
4672
5530
|
|
4673
|
-
@
|
4674
|
-
def exe(self) -> str:
|
4675
|
-
return os.path.join(check_not_none(self.root()), 'bin', 'pyenv')
|
5531
|
+
@async_cached_nullary
|
5532
|
+
async def exe(self) -> str:
|
5533
|
+
return os.path.join(check_not_none(await self.root()), 'bin', 'pyenv')
|
4676
5534
|
|
4677
|
-
def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
|
4678
|
-
if (root := self.root()) is None:
|
5535
|
+
async def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
|
5536
|
+
if (root := await self.root()) is None:
|
4679
5537
|
return []
|
4680
5538
|
ret = []
|
4681
5539
|
vp = os.path.join(root, 'versions')
|
@@ -4687,11 +5545,11 @@ class Pyenv:
|
|
4687
5545
|
ret.append((dn, ep))
|
4688
5546
|
return ret
|
4689
5547
|
|
4690
|
-
def installable_versions(self) -> ta.List[str]:
|
4691
|
-
if self.root() is None:
|
5548
|
+
async def installable_versions(self) -> ta.List[str]:
|
5549
|
+
if await self.root() is None:
|
4692
5550
|
return []
|
4693
5551
|
ret = []
|
4694
|
-
s =
|
5552
|
+
s = await asyncio_subprocess_check_output_str(await self.exe(), 'install', '--list')
|
4695
5553
|
for l in s.splitlines():
|
4696
5554
|
if not l.startswith(' '):
|
4697
5555
|
continue
|
@@ -4701,12 +5559,12 @@ class Pyenv:
|
|
4701
5559
|
ret.append(l)
|
4702
5560
|
return ret
|
4703
5561
|
|
4704
|
-
def update(self) -> bool:
|
4705
|
-
if (root := self.root()) is None:
|
5562
|
+
async def update(self) -> bool:
|
5563
|
+
if (root := await self.root()) is None:
|
4706
5564
|
return False
|
4707
5565
|
if not os.path.isdir(os.path.join(root, '.git')):
|
4708
5566
|
return False
|
4709
|
-
|
5567
|
+
await asyncio_subprocess_check_call('git', 'pull', cwd=root)
|
4710
5568
|
return True
|
4711
5569
|
|
4712
5570
|
|
@@ -4767,17 +5625,16 @@ THREADED_PYENV_INSTALL_OPTS = PyenvInstallOpts(conf_opts=['--disable-gil'])
|
|
4767
5625
|
|
4768
5626
|
class PyenvInstallOptsProvider(abc.ABC):
|
4769
5627
|
@abc.abstractmethod
|
4770
|
-
def opts(self) -> PyenvInstallOpts:
|
5628
|
+
def opts(self) -> ta.Awaitable[PyenvInstallOpts]:
|
4771
5629
|
raise NotImplementedError
|
4772
5630
|
|
4773
5631
|
|
4774
5632
|
class LinuxPyenvInstallOpts(PyenvInstallOptsProvider):
|
4775
|
-
def opts(self) -> PyenvInstallOpts:
|
5633
|
+
async def opts(self) -> PyenvInstallOpts:
|
4776
5634
|
return PyenvInstallOpts()
|
4777
5635
|
|
4778
5636
|
|
4779
5637
|
class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
4780
|
-
|
4781
5638
|
@cached_nullary
|
4782
5639
|
def framework_opts(self) -> PyenvInstallOpts:
|
4783
5640
|
return PyenvInstallOpts(conf_opts=['--enable-framework'])
|
@@ -4793,12 +5650,12 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
4793
5650
|
'zlib',
|
4794
5651
|
]
|
4795
5652
|
|
4796
|
-
@
|
4797
|
-
def brew_deps_opts(self) -> PyenvInstallOpts:
|
5653
|
+
@async_cached_nullary
|
5654
|
+
async def brew_deps_opts(self) -> PyenvInstallOpts:
|
4798
5655
|
cflags = []
|
4799
5656
|
ldflags = []
|
4800
5657
|
for dep in self.BREW_DEPS:
|
4801
|
-
dep_prefix =
|
5658
|
+
dep_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', dep)
|
4802
5659
|
cflags.append(f'-I{dep_prefix}/include')
|
4803
5660
|
ldflags.append(f'-L{dep_prefix}/lib')
|
4804
5661
|
return PyenvInstallOpts(
|
@@ -4806,13 +5663,13 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
4806
5663
|
ldflags=ldflags,
|
4807
5664
|
)
|
4808
5665
|
|
4809
|
-
@
|
4810
|
-
def brew_tcl_opts(self) -> PyenvInstallOpts:
|
4811
|
-
if
|
5666
|
+
@async_cached_nullary
|
5667
|
+
async def brew_tcl_opts(self) -> PyenvInstallOpts:
|
5668
|
+
if await asyncio_subprocess_try_output('brew', '--prefix', 'tcl-tk') is None:
|
4812
5669
|
return PyenvInstallOpts()
|
4813
5670
|
|
4814
|
-
tcl_tk_prefix =
|
4815
|
-
tcl_tk_ver_str =
|
5671
|
+
tcl_tk_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', 'tcl-tk')
|
5672
|
+
tcl_tk_ver_str = await asyncio_subprocess_check_output_str('brew', 'ls', '--versions', 'tcl-tk')
|
4816
5673
|
tcl_tk_ver = '.'.join(tcl_tk_ver_str.split()[1].split('.')[:2])
|
4817
5674
|
|
4818
5675
|
return PyenvInstallOpts(conf_opts=[
|
@@ -4827,11 +5684,11 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
4827
5684
|
# pkg_config_path += ':' + os.environ['PKG_CONFIG_PATH']
|
4828
5685
|
# return PyenvInstallOpts(env={'PKG_CONFIG_PATH': pkg_config_path})
|
4829
5686
|
|
4830
|
-
def opts(self) -> PyenvInstallOpts:
|
5687
|
+
async def opts(self) -> PyenvInstallOpts:
|
4831
5688
|
return PyenvInstallOpts().merge(
|
4832
5689
|
self.framework_opts(),
|
4833
|
-
self.brew_deps_opts(),
|
4834
|
-
self.brew_tcl_opts(),
|
5690
|
+
await self.brew_deps_opts(),
|
5691
|
+
await self.brew_tcl_opts(),
|
4835
5692
|
# self.brew_ssl_opts(),
|
4836
5693
|
)
|
4837
5694
|
|
@@ -4863,20 +5720,8 @@ class PyenvVersionInstaller:
|
|
4863
5720
|
) -> None:
|
4864
5721
|
super().__init__()
|
4865
5722
|
|
4866
|
-
if no_default_opts:
|
4867
|
-
if opts is None:
|
4868
|
-
opts = PyenvInstallOpts()
|
4869
|
-
else:
|
4870
|
-
lst = [opts if opts is not None else DEFAULT_PYENV_INSTALL_OPTS]
|
4871
|
-
if interp_opts.debug:
|
4872
|
-
lst.append(DEBUG_PYENV_INSTALL_OPTS)
|
4873
|
-
if interp_opts.threaded:
|
4874
|
-
lst.append(THREADED_PYENV_INSTALL_OPTS)
|
4875
|
-
lst.append(PLATFORM_PYENV_INSTALL_OPTS[sys.platform].opts())
|
4876
|
-
opts = PyenvInstallOpts().merge(*lst)
|
4877
|
-
|
4878
5723
|
self._version = version
|
4879
|
-
self.
|
5724
|
+
self._given_opts = opts
|
4880
5725
|
self._interp_opts = interp_opts
|
4881
5726
|
self._given_install_name = install_name
|
4882
5727
|
|
@@ -4887,9 +5732,21 @@ class PyenvVersionInstaller:
|
|
4887
5732
|
def version(self) -> str:
|
4888
5733
|
return self._version
|
4889
5734
|
|
4890
|
-
@
|
4891
|
-
def opts(self) -> PyenvInstallOpts:
|
4892
|
-
|
5735
|
+
@async_cached_nullary
|
5736
|
+
async def opts(self) -> PyenvInstallOpts:
|
5737
|
+
opts = self._given_opts
|
5738
|
+
if self._no_default_opts:
|
5739
|
+
if opts is None:
|
5740
|
+
opts = PyenvInstallOpts()
|
5741
|
+
else:
|
5742
|
+
lst = [self._given_opts if self._given_opts is not None else DEFAULT_PYENV_INSTALL_OPTS]
|
5743
|
+
if self._interp_opts.debug:
|
5744
|
+
lst.append(DEBUG_PYENV_INSTALL_OPTS)
|
5745
|
+
if self._interp_opts.threaded:
|
5746
|
+
lst.append(THREADED_PYENV_INSTALL_OPTS)
|
5747
|
+
lst.append(await PLATFORM_PYENV_INSTALL_OPTS[sys.platform].opts())
|
5748
|
+
opts = PyenvInstallOpts().merge(*lst)
|
5749
|
+
return opts
|
4893
5750
|
|
4894
5751
|
@cached_nullary
|
4895
5752
|
def install_name(self) -> str:
|
@@ -4897,17 +5754,18 @@ class PyenvVersionInstaller:
|
|
4897
5754
|
return self._given_install_name
|
4898
5755
|
return self._version + ('-debug' if self._interp_opts.debug else '')
|
4899
5756
|
|
4900
|
-
@
|
4901
|
-
def install_dir(self) -> str:
|
4902
|
-
return str(os.path.join(check_not_none(self._pyenv.root()), 'versions', self.install_name()))
|
5757
|
+
@async_cached_nullary
|
5758
|
+
async def install_dir(self) -> str:
|
5759
|
+
return str(os.path.join(check_not_none(await self._pyenv.root()), 'versions', self.install_name()))
|
4903
5760
|
|
4904
|
-
@
|
4905
|
-
def install(self) -> str:
|
4906
|
-
|
5761
|
+
@async_cached_nullary
|
5762
|
+
async def install(self) -> str:
|
5763
|
+
opts = await self.opts()
|
5764
|
+
env = {**os.environ, **opts.env}
|
4907
5765
|
for k, l in [
|
4908
|
-
('CFLAGS',
|
4909
|
-
('LDFLAGS',
|
4910
|
-
('PYTHON_CONFIGURE_OPTS',
|
5766
|
+
('CFLAGS', opts.cflags),
|
5767
|
+
('LDFLAGS', opts.ldflags),
|
5768
|
+
('PYTHON_CONFIGURE_OPTS', opts.conf_opts),
|
4911
5769
|
]:
|
4912
5770
|
v = ' '.join(l)
|
4913
5771
|
if k in os.environ:
|
@@ -4915,13 +5773,13 @@ class PyenvVersionInstaller:
|
|
4915
5773
|
env[k] = v
|
4916
5774
|
|
4917
5775
|
conf_args = [
|
4918
|
-
*
|
5776
|
+
*opts.opts,
|
4919
5777
|
self._version,
|
4920
5778
|
]
|
4921
5779
|
|
4922
5780
|
if self._given_install_name is not None:
|
4923
5781
|
full_args = [
|
4924
|
-
os.path.join(check_not_none(self._pyenv.root()), 'plugins', 'python-build', 'bin', 'python-build'),
|
5782
|
+
os.path.join(check_not_none(await self._pyenv.root()), 'plugins', 'python-build', 'bin', 'python-build'), # noqa
|
4925
5783
|
*conf_args,
|
4926
5784
|
self.install_dir(),
|
4927
5785
|
]
|
@@ -4932,12 +5790,12 @@ class PyenvVersionInstaller:
|
|
4932
5790
|
*conf_args,
|
4933
5791
|
]
|
4934
5792
|
|
4935
|
-
|
5793
|
+
await asyncio_subprocess_check_call(
|
4936
5794
|
*full_args,
|
4937
5795
|
env=env,
|
4938
5796
|
)
|
4939
5797
|
|
4940
|
-
exe = os.path.join(self.install_dir(), 'bin', 'python')
|
5798
|
+
exe = os.path.join(await self.install_dir(), 'bin', 'python')
|
4941
5799
|
if not os.path.isfile(exe):
|
4942
5800
|
raise RuntimeError(f'Interpreter not found: {exe}')
|
4943
5801
|
return exe
|
@@ -4947,7 +5805,6 @@ class PyenvVersionInstaller:
|
|
4947
5805
|
|
4948
5806
|
|
4949
5807
|
class PyenvInterpProvider(InterpProvider):
|
4950
|
-
|
4951
5808
|
def __init__(
|
4952
5809
|
self,
|
4953
5810
|
pyenv: Pyenv = Pyenv(),
|
@@ -4990,11 +5847,11 @@ class PyenvInterpProvider(InterpProvider):
|
|
4990
5847
|
exe: str
|
4991
5848
|
version: InterpVersion
|
4992
5849
|
|
4993
|
-
def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
|
5850
|
+
async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
|
4994
5851
|
iv: ta.Optional[InterpVersion]
|
4995
5852
|
if self._inspect:
|
4996
5853
|
try:
|
4997
|
-
iv = check_not_none(self._inspector.inspect(ep)).iv
|
5854
|
+
iv = check_not_none(await self._inspector.inspect(ep)).iv
|
4998
5855
|
except Exception as e: # noqa
|
4999
5856
|
return None
|
5000
5857
|
else:
|
@@ -5007,10 +5864,10 @@ class PyenvInterpProvider(InterpProvider):
|
|
5007
5864
|
version=iv,
|
5008
5865
|
)
|
5009
5866
|
|
5010
|
-
def installed(self) -> ta.Sequence[Installed]:
|
5867
|
+
async def installed(self) -> ta.Sequence[Installed]:
|
5011
5868
|
ret: ta.List[PyenvInterpProvider.Installed] = []
|
5012
|
-
for vn, ep in self._pyenv.version_exes():
|
5013
|
-
if (i := self._make_installed(vn, ep)) is None:
|
5869
|
+
for vn, ep in await self._pyenv.version_exes():
|
5870
|
+
if (i := await self._make_installed(vn, ep)) is None:
|
5014
5871
|
log.debug('Invalid pyenv version: %s', vn)
|
5015
5872
|
continue
|
5016
5873
|
ret.append(i)
|
@@ -5018,11 +5875,11 @@ class PyenvInterpProvider(InterpProvider):
|
|
5018
5875
|
|
5019
5876
|
#
|
5020
5877
|
|
5021
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5022
|
-
return [i.version for i in self.installed()]
|
5878
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5879
|
+
return [i.version for i in await self.installed()]
|
5023
5880
|
|
5024
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5025
|
-
for i in self.installed():
|
5881
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
5882
|
+
for i in await self.installed():
|
5026
5883
|
if i.version == version:
|
5027
5884
|
return Interp(
|
5028
5885
|
exe=i.exe,
|
@@ -5032,10 +5889,10 @@ class PyenvInterpProvider(InterpProvider):
|
|
5032
5889
|
|
5033
5890
|
#
|
5034
5891
|
|
5035
|
-
def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5892
|
+
async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5036
5893
|
lst = []
|
5037
5894
|
|
5038
|
-
for vs in self._pyenv.installable_versions():
|
5895
|
+
for vs in await self._pyenv.installable_versions():
|
5039
5896
|
if (iv := self.guess_version(vs)) is None:
|
5040
5897
|
continue
|
5041
5898
|
if iv.opts.debug:
|
@@ -5045,16 +5902,16 @@ class PyenvInterpProvider(InterpProvider):
|
|
5045
5902
|
|
5046
5903
|
return lst
|
5047
5904
|
|
5048
|
-
def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5049
|
-
lst = self._get_installable_versions(spec)
|
5905
|
+
async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5906
|
+
lst = await self._get_installable_versions(spec)
|
5050
5907
|
|
5051
5908
|
if self._try_update and not any(v in spec for v in lst):
|
5052
5909
|
if self._pyenv.update():
|
5053
|
-
lst = self._get_installable_versions(spec)
|
5910
|
+
lst = await self._get_installable_versions(spec)
|
5054
5911
|
|
5055
5912
|
return lst
|
5056
5913
|
|
5057
|
-
def install_version(self, version: InterpVersion) -> Interp:
|
5914
|
+
async def install_version(self, version: InterpVersion) -> Interp:
|
5058
5915
|
inst_version = str(version.version)
|
5059
5916
|
inst_opts = version.opts
|
5060
5917
|
if inst_opts.threaded:
|
@@ -5066,7 +5923,7 @@ class PyenvInterpProvider(InterpProvider):
|
|
5066
5923
|
interp_opts=inst_opts,
|
5067
5924
|
)
|
5068
5925
|
|
5069
|
-
exe = installer.install()
|
5926
|
+
exe = await installer.install()
|
5070
5927
|
return Interp(exe, version)
|
5071
5928
|
|
5072
5929
|
|
@@ -5145,7 +6002,7 @@ class SystemInterpProvider(InterpProvider):
|
|
5145
6002
|
|
5146
6003
|
#
|
5147
6004
|
|
5148
|
-
def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
|
6005
|
+
async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
|
5149
6006
|
if not self.inspect:
|
5150
6007
|
s = os.path.basename(exe)
|
5151
6008
|
if s.startswith('python'):
|
@@ -5155,13 +6012,13 @@ class SystemInterpProvider(InterpProvider):
|
|
5155
6012
|
return InterpVersion.parse(s)
|
5156
6013
|
except InvalidVersion:
|
5157
6014
|
pass
|
5158
|
-
ii = self.inspector.inspect(exe)
|
6015
|
+
ii = await self.inspector.inspect(exe)
|
5159
6016
|
return ii.iv if ii is not None else None
|
5160
6017
|
|
5161
|
-
def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
|
6018
|
+
async def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
|
5162
6019
|
lst = []
|
5163
6020
|
for e in self.exes():
|
5164
|
-
if (ev := self.get_exe_version(e)) is None:
|
6021
|
+
if (ev := await self.get_exe_version(e)) is None:
|
5165
6022
|
log.debug('Invalid system version: %s', e)
|
5166
6023
|
continue
|
5167
6024
|
lst.append((e, ev))
|
@@ -5169,11 +6026,11 @@ class SystemInterpProvider(InterpProvider):
|
|
5169
6026
|
|
5170
6027
|
#
|
5171
6028
|
|
5172
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5173
|
-
return [ev for e, ev in self.exe_versions()]
|
6029
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
6030
|
+
return [ev for e, ev in await self.exe_versions()]
|
5174
6031
|
|
5175
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5176
|
-
for e, ev in self.exe_versions():
|
6032
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
6033
|
+
for e, ev in await self.exe_versions():
|
5177
6034
|
if ev != version:
|
5178
6035
|
continue
|
5179
6036
|
return Interp(
|
@@ -5194,9 +6051,11 @@ def bind_remote(
|
|
5194
6051
|
lst: ta.List[InjectorBindingOrBindings] = [
|
5195
6052
|
inj.bind(remote_config),
|
5196
6053
|
|
5197
|
-
inj.bind(
|
6054
|
+
inj.bind(SubprocessRemoteSpawning, singleton=True),
|
6055
|
+
inj.bind(RemoteSpawning, to_key=SubprocessRemoteSpawning),
|
5198
6056
|
|
5199
|
-
inj.bind(
|
6057
|
+
inj.bind(PyremoteRemoteExecutionConnector, singleton=True),
|
6058
|
+
inj.bind(RemoteExecutionConnector, to_key=PyremoteRemoteExecutionConnector),
|
5200
6059
|
]
|
5201
6060
|
|
5202
6061
|
if (pf := remote_config.payload_file) is not None:
|
@@ -5220,13 +6079,14 @@ class InterpResolver:
|
|
5220
6079
|
providers: ta.Sequence[ta.Tuple[str, InterpProvider]],
|
5221
6080
|
) -> None:
|
5222
6081
|
super().__init__()
|
6082
|
+
|
5223
6083
|
self._providers: ta.Mapping[str, InterpProvider] = collections.OrderedDict(providers)
|
5224
6084
|
|
5225
|
-
def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
|
6085
|
+
async def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
|
5226
6086
|
lst = [
|
5227
6087
|
(i, si)
|
5228
6088
|
for i, p in enumerate(self._providers.values())
|
5229
|
-
for si in p.get_installed_versions(spec)
|
6089
|
+
for si in await p.get_installed_versions(spec)
|
5230
6090
|
if spec.contains(si)
|
5231
6091
|
]
|
5232
6092
|
|
@@ -5238,16 +6098,16 @@ class InterpResolver:
|
|
5238
6098
|
bp = list(self._providers.values())[bi]
|
5239
6099
|
return (bp, bv)
|
5240
6100
|
|
5241
|
-
def resolve(
|
6101
|
+
async def resolve(
|
5242
6102
|
self,
|
5243
6103
|
spec: InterpSpecifier,
|
5244
6104
|
*,
|
5245
6105
|
install: bool = False,
|
5246
6106
|
) -> ta.Optional[Interp]:
|
5247
|
-
tup = self._resolve_installed(spec)
|
6107
|
+
tup = await self._resolve_installed(spec)
|
5248
6108
|
if tup is not None:
|
5249
6109
|
bp, bv = tup
|
5250
|
-
return bp.get_installed_version(bv)
|
6110
|
+
return await bp.get_installed_version(bv)
|
5251
6111
|
|
5252
6112
|
if not install:
|
5253
6113
|
return None
|
@@ -5255,21 +6115,21 @@ class InterpResolver:
|
|
5255
6115
|
tp = list(self._providers.values())[0] # noqa
|
5256
6116
|
|
5257
6117
|
sv = sorted(
|
5258
|
-
[s for s in tp.get_installable_versions(spec) if s in spec],
|
6118
|
+
[s for s in await tp.get_installable_versions(spec) if s in spec],
|
5259
6119
|
key=lambda s: s.version,
|
5260
6120
|
)
|
5261
6121
|
if not sv:
|
5262
6122
|
return None
|
5263
6123
|
|
5264
6124
|
bv = sv[-1]
|
5265
|
-
return tp.install_version(bv)
|
6125
|
+
return await tp.install_version(bv)
|
5266
6126
|
|
5267
|
-
def list(self, spec: InterpSpecifier) -> None:
|
6127
|
+
async def list(self, spec: InterpSpecifier) -> None:
|
5268
6128
|
print('installed:')
|
5269
6129
|
for n, p in self._providers.items():
|
5270
6130
|
lst = [
|
5271
6131
|
si
|
5272
|
-
for si in p.get_installed_versions(spec)
|
6132
|
+
for si in await p.get_installed_versions(spec)
|
5273
6133
|
if spec.contains(si)
|
5274
6134
|
]
|
5275
6135
|
if lst:
|
@@ -5283,7 +6143,7 @@ class InterpResolver:
|
|
5283
6143
|
for n, p in self._providers.items():
|
5284
6144
|
lst = [
|
5285
6145
|
si
|
5286
|
-
for si in p.get_installable_versions(spec)
|
6146
|
+
for si in await p.get_installable_versions(spec)
|
5287
6147
|
if spec.contains(si)
|
5288
6148
|
]
|
5289
6149
|
if lst:
|
@@ -5325,9 +6185,9 @@ class InterpCommand(Command['InterpCommand.Output']):
|
|
5325
6185
|
|
5326
6186
|
|
5327
6187
|
class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
|
5328
|
-
def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
|
6188
|
+
async def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
|
5329
6189
|
i = InterpSpecifier.parse(check_not_none(cmd.spec))
|
5330
|
-
o = check_not_none(DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
|
6190
|
+
o = check_not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
|
5331
6191
|
return InterpCommand.Output(
|
5332
6192
|
exe=o.exe,
|
5333
6193
|
version=str(o.version.version),
|
@@ -5366,7 +6226,7 @@ def bind_command(
|
|
5366
6226
|
class _FactoryCommandExecutor(CommandExecutor):
|
5367
6227
|
factory: ta.Callable[[], CommandExecutor]
|
5368
6228
|
|
5369
|
-
def execute(self, i: Command) -> Command.Output:
|
6229
|
+
def execute(self, i: Command) -> ta.Awaitable[Command.Output]:
|
5370
6230
|
return self.factory().execute(i)
|
5371
6231
|
|
5372
6232
|
|
@@ -5521,31 +6381,7 @@ def main_bootstrap(bs: MainBootstrap) -> Injector:
|
|
5521
6381
|
##
|
5522
6382
|
|
5523
6383
|
|
5524
|
-
def
|
5525
|
-
import argparse
|
5526
|
-
|
5527
|
-
parser = argparse.ArgumentParser()
|
5528
|
-
|
5529
|
-
parser.add_argument('--_payload-file')
|
5530
|
-
|
5531
|
-
parser.add_argument('-s', '--shell')
|
5532
|
-
parser.add_argument('-q', '--shell-quote', action='store_true')
|
5533
|
-
parser.add_argument('--python', default='python3')
|
5534
|
-
|
5535
|
-
parser.add_argument('--pycharm-debug-port', type=int)
|
5536
|
-
parser.add_argument('--pycharm-debug-host')
|
5537
|
-
parser.add_argument('--pycharm-debug-version')
|
5538
|
-
|
5539
|
-
parser.add_argument('--debug', action='store_true')
|
5540
|
-
|
5541
|
-
parser.add_argument('--local', action='store_true')
|
5542
|
-
|
5543
|
-
parser.add_argument('command', nargs='+')
|
5544
|
-
|
5545
|
-
args = parser.parse_args()
|
5546
|
-
|
5547
|
-
#
|
5548
|
-
|
6384
|
+
async def _async_main(args: ta.Any) -> None:
|
5549
6385
|
bs = MainBootstrap(
|
5550
6386
|
main_config=MainConfig(
|
5551
6387
|
log_level='DEBUG' if args.debug else 'INFO',
|
@@ -5558,12 +6394,16 @@ def _main() -> None:
|
|
5558
6394
|
|
5559
6395
|
pycharm_remote_debug=PycharmRemoteDebug(
|
5560
6396
|
port=args.pycharm_debug_port,
|
5561
|
-
host=args.pycharm_debug_host,
|
6397
|
+
**(dict(host=args.pycharm_debug_host) if args.pycharm_debug_host is not None else {}),
|
5562
6398
|
install_version=args.pycharm_debug_version,
|
5563
6399
|
) if args.pycharm_debug_port is not None else None,
|
6400
|
+
|
6401
|
+
timebomb_delay_s=args.remote_timebomb_delay_s,
|
5564
6402
|
),
|
5565
6403
|
)
|
5566
6404
|
|
6405
|
+
#
|
6406
|
+
|
5567
6407
|
injector = main_bootstrap(
|
5568
6408
|
bs,
|
5569
6409
|
)
|
@@ -5582,7 +6422,7 @@ def _main() -> None:
|
|
5582
6422
|
|
5583
6423
|
#
|
5584
6424
|
|
5585
|
-
with contextlib.
|
6425
|
+
async with contextlib.AsyncExitStack() as es:
|
5586
6426
|
ce: CommandExecutor
|
5587
6427
|
|
5588
6428
|
if args.local:
|
@@ -5595,16 +6435,51 @@ def _main() -> None:
|
|
5595
6435
|
python=args.python,
|
5596
6436
|
)
|
5597
6437
|
|
5598
|
-
ce = es.
|
6438
|
+
ce = await es.enter_async_context(injector[RemoteExecutionConnector].connect(tgt, bs)) # noqa
|
5599
6439
|
|
5600
|
-
|
5601
|
-
|
6440
|
+
async def run_command(cmd: Command) -> None:
|
6441
|
+
res = await ce.try_execute(
|
5602
6442
|
cmd,
|
5603
6443
|
log=log,
|
5604
6444
|
omit_exc_object=True,
|
5605
6445
|
)
|
5606
6446
|
|
5607
|
-
print(msh.marshal_obj(
|
6447
|
+
print(msh.marshal_obj(res, opts=ObjMarshalOptions(raw_bytes=True)))
|
6448
|
+
|
6449
|
+
await asyncio.gather(*[
|
6450
|
+
run_command(cmd)
|
6451
|
+
for cmd in cmds
|
6452
|
+
])
|
6453
|
+
|
6454
|
+
|
6455
|
+
def _main() -> None:
|
6456
|
+
import argparse
|
6457
|
+
|
6458
|
+
parser = argparse.ArgumentParser()
|
6459
|
+
|
6460
|
+
parser.add_argument('--_payload-file')
|
6461
|
+
|
6462
|
+
parser.add_argument('-s', '--shell')
|
6463
|
+
parser.add_argument('-q', '--shell-quote', action='store_true')
|
6464
|
+
parser.add_argument('--python', default='python3')
|
6465
|
+
|
6466
|
+
parser.add_argument('--pycharm-debug-port', type=int)
|
6467
|
+
parser.add_argument('--pycharm-debug-host')
|
6468
|
+
parser.add_argument('--pycharm-debug-version')
|
6469
|
+
|
6470
|
+
parser.add_argument('--remote-timebomb-delay-s', type=float)
|
6471
|
+
|
6472
|
+
parser.add_argument('--debug', action='store_true')
|
6473
|
+
|
6474
|
+
parser.add_argument('--local', action='store_true')
|
6475
|
+
|
6476
|
+
parser.add_argument('command', nargs='+')
|
6477
|
+
|
6478
|
+
args = parser.parse_args()
|
6479
|
+
|
6480
|
+
#
|
6481
|
+
|
6482
|
+
asyncio.run(_async_main(args))
|
5608
6483
|
|
5609
6484
|
|
5610
6485
|
if __name__ == '__main__':
|