ominfra 0.0.0.dev146__py3-none-any.whl → 0.0.0.dev148__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/configs.py +30 -5
- ominfra/manage/commands/base.py +5 -5
- ominfra/manage/commands/execution.py +2 -2
- ominfra/manage/commands/inject.py +1 -1
- ominfra/manage/commands/interp.py +2 -2
- ominfra/manage/commands/subprocess.py +22 -14
- ominfra/manage/deploy/command.py +1 -1
- ominfra/manage/main.py +48 -32
- ominfra/manage/remote/_main.py +172 -0
- ominfra/manage/remote/channel.py +41 -16
- ominfra/manage/remote/config.py +10 -0
- ominfra/manage/remote/connection.py +106 -0
- ominfra/manage/remote/execution.py +244 -155
- ominfra/manage/remote/inject.py +7 -3
- ominfra/manage/remote/spawning.py +51 -33
- ominfra/pyremote.py +28 -3
- ominfra/scripts/journald2aws.py +127 -28
- ominfra/scripts/manage.py +1361 -487
- ominfra/scripts/supervisor.py +59 -10
- {ominfra-0.0.0.dev146.dist-info → ominfra-0.0.0.dev148.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev146.dist-info → ominfra-0.0.0.dev148.dist-info}/RECORD +25 -23
- {ominfra-0.0.0.dev146.dist-info → ominfra-0.0.0.dev148.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev146.dist-info → ominfra-0.0.0.dev148.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev146.dist-info → ominfra-0.0.0.dev148.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev146.dist-info → ominfra-0.0.0.dev148.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -3,16 +3,20 @@
|
|
3
3
|
# @omlish-lite
|
4
4
|
# @omlish-script
|
5
5
|
# @omlish-amalg-output ../manage/main.py
|
6
|
-
# ruff: noqa: N802 UP006 UP007 UP036
|
6
|
+
# ruff: noqa: N802 TC003 UP006 UP007 UP036
|
7
7
|
"""
|
8
8
|
manage.py -s 'docker run -i python:3.12'
|
9
9
|
manage.py -s 'ssh -i /foo/bar.pem foo@bar.baz' -q --python=python3.8
|
10
10
|
"""
|
11
11
|
import abc
|
12
|
+
import asyncio
|
13
|
+
import asyncio.base_subprocess
|
14
|
+
import asyncio.subprocess
|
12
15
|
import base64
|
13
16
|
import collections
|
14
17
|
import collections.abc
|
15
18
|
import contextlib
|
19
|
+
import ctypes as ct
|
16
20
|
import dataclasses as dc
|
17
21
|
import datetime
|
18
22
|
import decimal
|
@@ -30,6 +34,7 @@ import pwd
|
|
30
34
|
import re
|
31
35
|
import shlex
|
32
36
|
import shutil
|
37
|
+
import signal
|
33
38
|
import site
|
34
39
|
import struct
|
35
40
|
import subprocess
|
@@ -62,6 +67,9 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
|
|
62
67
|
VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType] # noqa
|
63
68
|
VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
|
64
69
|
|
70
|
+
# ../../omlish/lite/asyncio/asyncio.py
|
71
|
+
AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
|
72
|
+
|
65
73
|
# ../../omlish/lite/cached.py
|
66
74
|
T = ta.TypeVar('T')
|
67
75
|
CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
|
@@ -822,10 +830,8 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
822
830
|
with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
|
823
831
|
main_src = sf.read()
|
824
832
|
|
825
|
-
# Restore
|
833
|
+
# Restore vars
|
826
834
|
sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
|
827
|
-
|
828
|
-
# Grab context name
|
829
835
|
context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
|
830
836
|
|
831
837
|
# Write third ack
|
@@ -1000,12 +1006,102 @@ class PyremoteBootstrapDriver:
|
|
1000
1006
|
else:
|
1001
1007
|
raise TypeError(go)
|
1002
1008
|
|
1009
|
+
async def async_run(
|
1010
|
+
self,
|
1011
|
+
input: ta.Any, # asyncio.StreamWriter # noqa
|
1012
|
+
output: ta.Any, # asyncio.StreamReader
|
1013
|
+
) -> Result:
|
1014
|
+
gen = self.gen()
|
1015
|
+
|
1016
|
+
gi: ta.Optional[bytes] = None
|
1017
|
+
while True:
|
1018
|
+
try:
|
1019
|
+
if gi is not None:
|
1020
|
+
go = gen.send(gi)
|
1021
|
+
else:
|
1022
|
+
go = next(gen)
|
1023
|
+
except StopIteration as e:
|
1024
|
+
return e.value
|
1025
|
+
|
1026
|
+
if isinstance(go, self.Read):
|
1027
|
+
if len(gi := await input.read(go.sz)) != go.sz:
|
1028
|
+
raise EOFError
|
1029
|
+
elif isinstance(go, self.Write):
|
1030
|
+
gi = None
|
1031
|
+
output.write(go.d)
|
1032
|
+
await output.drain()
|
1033
|
+
else:
|
1034
|
+
raise TypeError(go)
|
1035
|
+
|
1036
|
+
|
1037
|
+
########################################
|
1038
|
+
# ../../../omlish/lite/asyncio/asyncio.py
|
1039
|
+
|
1040
|
+
|
1041
|
+
##
|
1042
|
+
|
1043
|
+
|
1044
|
+
ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
|
1045
|
+
|
1046
|
+
|
1047
|
+
async def asyncio_open_stream_reader(
|
1048
|
+
f: ta.IO,
|
1049
|
+
loop: ta.Any = None,
|
1050
|
+
*,
|
1051
|
+
limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
|
1052
|
+
) -> asyncio.StreamReader:
|
1053
|
+
if loop is None:
|
1054
|
+
loop = asyncio.get_running_loop()
|
1055
|
+
|
1056
|
+
reader = asyncio.StreamReader(limit=limit, loop=loop)
|
1057
|
+
await loop.connect_read_pipe(
|
1058
|
+
lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
|
1059
|
+
f,
|
1060
|
+
)
|
1061
|
+
|
1062
|
+
return reader
|
1063
|
+
|
1064
|
+
|
1065
|
+
async def asyncio_open_stream_writer(
|
1066
|
+
f: ta.IO,
|
1067
|
+
loop: ta.Any = None,
|
1068
|
+
) -> asyncio.StreamWriter:
|
1069
|
+
if loop is None:
|
1070
|
+
loop = asyncio.get_running_loop()
|
1071
|
+
|
1072
|
+
writer_transport, writer_protocol = await loop.connect_write_pipe(
|
1073
|
+
lambda: asyncio.streams.FlowControlMixin(loop=loop),
|
1074
|
+
f,
|
1075
|
+
)
|
1076
|
+
|
1077
|
+
return asyncio.streams.StreamWriter(
|
1078
|
+
writer_transport,
|
1079
|
+
writer_protocol,
|
1080
|
+
None,
|
1081
|
+
loop,
|
1082
|
+
)
|
1083
|
+
|
1084
|
+
|
1085
|
+
##
|
1086
|
+
|
1087
|
+
|
1088
|
+
def asyncio_maybe_timeout(
|
1089
|
+
fut: AwaitableT,
|
1090
|
+
timeout: ta.Optional[float] = None,
|
1091
|
+
) -> AwaitableT:
|
1092
|
+
if timeout is not None:
|
1093
|
+
fut = asyncio.wait_for(fut, timeout) # type: ignore
|
1094
|
+
return fut
|
1095
|
+
|
1003
1096
|
|
1004
1097
|
########################################
|
1005
1098
|
# ../../../omlish/lite/cached.py
|
1006
1099
|
|
1007
1100
|
|
1008
|
-
|
1101
|
+
##
|
1102
|
+
|
1103
|
+
|
1104
|
+
class _AbstractCachedNullary:
|
1009
1105
|
def __init__(self, fn):
|
1010
1106
|
super().__init__()
|
1011
1107
|
self._fn = fn
|
@@ -1013,17 +1109,25 @@ class _cached_nullary: # noqa
|
|
1013
1109
|
functools.update_wrapper(self, fn)
|
1014
1110
|
|
1015
1111
|
def __call__(self, *args, **kwargs): # noqa
|
1016
|
-
|
1017
|
-
self._value = self._fn()
|
1018
|
-
return self._value
|
1112
|
+
raise TypeError
|
1019
1113
|
|
1020
1114
|
def __get__(self, instance, owner): # noqa
|
1021
1115
|
bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
|
1022
1116
|
return bound
|
1023
1117
|
|
1024
1118
|
|
1119
|
+
##
|
1120
|
+
|
1121
|
+
|
1122
|
+
class _CachedNullary(_AbstractCachedNullary):
|
1123
|
+
def __call__(self, *args, **kwargs): # noqa
|
1124
|
+
if self._value is self._missing:
|
1125
|
+
self._value = self._fn()
|
1126
|
+
return self._value
|
1127
|
+
|
1128
|
+
|
1025
1129
|
def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
1026
|
-
return
|
1130
|
+
return _CachedNullary(fn)
|
1027
1131
|
|
1028
1132
|
|
1029
1133
|
def static_init(fn: CallableT) -> CallableT:
|
@@ -1032,6 +1136,20 @@ def static_init(fn: CallableT) -> CallableT:
|
|
1032
1136
|
return fn
|
1033
1137
|
|
1034
1138
|
|
1139
|
+
##
|
1140
|
+
|
1141
|
+
|
1142
|
+
class _AsyncCachedNullary(_AbstractCachedNullary):
|
1143
|
+
async def __call__(self, *args, **kwargs):
|
1144
|
+
if self._value is self._missing:
|
1145
|
+
self._value = await self._fn()
|
1146
|
+
return self._value
|
1147
|
+
|
1148
|
+
|
1149
|
+
def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
1150
|
+
return _AsyncCachedNullary(fn)
|
1151
|
+
|
1152
|
+
|
1035
1153
|
########################################
|
1036
1154
|
# ../../../omlish/lite/check.py
|
1037
1155
|
|
@@ -1129,6 +1247,30 @@ def check_non_empty(v: SizedT) -> SizedT:
|
|
1129
1247
|
return v
|
1130
1248
|
|
1131
1249
|
|
1250
|
+
########################################
|
1251
|
+
# ../../../omlish/lite/deathsig.py
|
1252
|
+
|
1253
|
+
|
1254
|
+
LINUX_PR_SET_PDEATHSIG = 1 # Second arg is a signal
|
1255
|
+
LINUX_PR_GET_PDEATHSIG = 2 # Second arg is a ptr to return the signal
|
1256
|
+
|
1257
|
+
|
1258
|
+
def set_process_deathsig(sig: int) -> bool:
|
1259
|
+
if sys.platform == 'linux':
|
1260
|
+
libc = ct.CDLL('libc.so.6')
|
1261
|
+
|
1262
|
+
# int prctl(int option, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5);
|
1263
|
+
libc.prctl.restype = ct.c_int
|
1264
|
+
libc.prctl.argtypes = [ct.c_int, ct.c_ulong, ct.c_ulong, ct.c_ulong, ct.c_ulong]
|
1265
|
+
|
1266
|
+
libc.prctl(LINUX_PR_SET_PDEATHSIG, sig, 0, 0, 0, 0)
|
1267
|
+
|
1268
|
+
return True
|
1269
|
+
|
1270
|
+
else:
|
1271
|
+
return False
|
1272
|
+
|
1273
|
+
|
1132
1274
|
########################################
|
1133
1275
|
# ../../../omlish/lite/json.py
|
1134
1276
|
|
@@ -1241,7 +1383,6 @@ def pycharm_debug_connect(prd: PycharmRemoteDebug) -> None:
|
|
1241
1383
|
def pycharm_debug_preamble(prd: PycharmRemoteDebug) -> str:
|
1242
1384
|
import inspect
|
1243
1385
|
import textwrap
|
1244
|
-
|
1245
1386
|
return textwrap.dedent(f"""
|
1246
1387
|
{inspect.getsource(pycharm_debug_connect)}
|
1247
1388
|
|
@@ -1911,8 +2052,8 @@ class Command(abc.ABC, ta.Generic[CommandOutputT]):
|
|
1911
2052
|
pass
|
1912
2053
|
|
1913
2054
|
@ta.final
|
1914
|
-
def execute(self, executor: 'CommandExecutor') -> CommandOutputT:
|
1915
|
-
return check_isinstance(executor.execute(self), self.Output) # type: ignore[return-value]
|
2055
|
+
async def execute(self, executor: 'CommandExecutor') -> CommandOutputT:
|
2056
|
+
return check_isinstance(await executor.execute(self), self.Output) # type: ignore[return-value]
|
1916
2057
|
|
1917
2058
|
|
1918
2059
|
##
|
@@ -1973,10 +2114,10 @@ class CommandOutputOrExceptionData(CommandOutputOrException):
|
|
1973
2114
|
|
1974
2115
|
class CommandExecutor(abc.ABC, ta.Generic[CommandT, CommandOutputT]):
|
1975
2116
|
@abc.abstractmethod
|
1976
|
-
def execute(self, cmd: CommandT) -> CommandOutputT:
|
2117
|
+
def execute(self, cmd: CommandT) -> ta.Awaitable[CommandOutputT]:
|
1977
2118
|
raise NotImplementedError
|
1978
2119
|
|
1979
|
-
def try_execute(
|
2120
|
+
async def try_execute(
|
1980
2121
|
self,
|
1981
2122
|
cmd: CommandT,
|
1982
2123
|
*,
|
@@ -1984,7 +2125,7 @@ class CommandExecutor(abc.ABC, ta.Generic[CommandT, CommandOutputT]):
|
|
1984
2125
|
omit_exc_object: bool = False,
|
1985
2126
|
) -> CommandOutputOrException[CommandOutputT]:
|
1986
2127
|
try:
|
1987
|
-
o = self.execute(cmd)
|
2128
|
+
o = await self.execute(cmd)
|
1988
2129
|
|
1989
2130
|
except Exception as e: # noqa
|
1990
2131
|
if log is not None:
|
@@ -2055,8 +2196,18 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
|
|
2055
2196
|
class RemoteConfig:
|
2056
2197
|
payload_file: ta.Optional[str] = None
|
2057
2198
|
|
2199
|
+
set_pgid: bool = True
|
2200
|
+
|
2201
|
+
deathsig: ta.Optional[str] = 'KILL'
|
2202
|
+
|
2058
2203
|
pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
|
2059
2204
|
|
2205
|
+
forward_logging: bool = True
|
2206
|
+
|
2207
|
+
timebomb_delay_s: ta.Optional[float] = 60 * 60.
|
2208
|
+
|
2209
|
+
heartbeat_interval_s: float = 3.
|
2210
|
+
|
2060
2211
|
|
2061
2212
|
########################################
|
2062
2213
|
# ../remote/payload.py
|
@@ -3789,6 +3940,8 @@ class InterpSpecifier:
|
|
3789
3940
|
s, o = InterpOpts.parse_suffix(s)
|
3790
3941
|
if not any(s.startswith(o) for o in Specifier.OPERATORS):
|
3791
3942
|
s = '~=' + s
|
3943
|
+
if s.count('.') < 2:
|
3944
|
+
s += '.0'
|
3792
3945
|
return cls(
|
3793
3946
|
specifier=Specifier(s),
|
3794
3947
|
opts=o,
|
@@ -3835,9 +3988,9 @@ class LocalCommandExecutor(CommandExecutor):
|
|
3835
3988
|
|
3836
3989
|
self._command_executors = command_executors
|
3837
3990
|
|
3838
|
-
def execute(self, cmd: Command) -> Command.Output:
|
3991
|
+
async def execute(self, cmd: Command) -> Command.Output:
|
3839
3992
|
ce: CommandExecutor = self._command_executors[type(cmd)]
|
3840
|
-
return ce.execute(cmd)
|
3993
|
+
return await ce.execute(cmd)
|
3841
3994
|
|
3842
3995
|
|
3843
3996
|
########################################
|
@@ -3883,7 +4036,7 @@ class DeployCommand(Command['DeployCommand.Output']):
|
|
3883
4036
|
|
3884
4037
|
|
3885
4038
|
class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
|
3886
|
-
def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
4039
|
+
async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
3887
4040
|
log.info('Deploying!')
|
3888
4041
|
|
3889
4042
|
return DeployCommand.Output()
|
@@ -3905,11 +4058,30 @@ ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMar
|
|
3905
4058
|
# ../remote/channel.py
|
3906
4059
|
|
3907
4060
|
|
3908
|
-
|
4061
|
+
##
|
4062
|
+
|
4063
|
+
|
4064
|
+
class RemoteChannel(abc.ABC):
|
4065
|
+
@abc.abstractmethod
|
4066
|
+
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Awaitable[None]:
|
4067
|
+
raise NotImplementedError
|
4068
|
+
|
4069
|
+
@abc.abstractmethod
|
4070
|
+
def recv_obj(self, ty: ta.Type[T]) -> ta.Awaitable[ta.Optional[T]]:
|
4071
|
+
raise NotImplementedError
|
4072
|
+
|
4073
|
+
def set_marshaler(self, msh: ObjMarshalerManager) -> None: # noqa
|
4074
|
+
pass
|
4075
|
+
|
4076
|
+
|
4077
|
+
##
|
4078
|
+
|
4079
|
+
|
4080
|
+
class RemoteChannelImpl(RemoteChannel):
|
3909
4081
|
def __init__(
|
3910
4082
|
self,
|
3911
|
-
input:
|
3912
|
-
output:
|
4083
|
+
input: asyncio.StreamReader, # noqa
|
4084
|
+
output: asyncio.StreamWriter,
|
3913
4085
|
*,
|
3914
4086
|
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
3915
4087
|
) -> None:
|
@@ -3919,41 +4091,46 @@ class RemoteChannel:
|
|
3919
4091
|
self._output = output
|
3920
4092
|
self._msh = msh
|
3921
4093
|
|
3922
|
-
self.
|
4094
|
+
self._input_lock = asyncio.Lock()
|
4095
|
+
self._output_lock = asyncio.Lock()
|
3923
4096
|
|
3924
4097
|
def set_marshaler(self, msh: ObjMarshalerManager) -> None:
|
3925
4098
|
self._msh = msh
|
3926
4099
|
|
3927
|
-
|
4100
|
+
#
|
4101
|
+
|
4102
|
+
async def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
3928
4103
|
j = json_dumps_compact(self._msh.marshal_obj(o, ty))
|
3929
4104
|
d = j.encode('utf-8')
|
3930
4105
|
|
3931
4106
|
self._output.write(struct.pack('<I', len(d)))
|
3932
4107
|
self._output.write(d)
|
3933
|
-
self._output.
|
4108
|
+
await self._output.drain()
|
3934
4109
|
|
3935
|
-
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
3936
|
-
with self.
|
3937
|
-
return self._send_obj(o, ty)
|
4110
|
+
async def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
4111
|
+
async with self._output_lock:
|
4112
|
+
return await self._send_obj(o, ty)
|
3938
4113
|
|
3939
|
-
|
3940
|
-
|
4114
|
+
#
|
4115
|
+
|
4116
|
+
async def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
4117
|
+
d = await self._input.read(4)
|
3941
4118
|
if not d:
|
3942
4119
|
return None
|
3943
4120
|
if len(d) != 4:
|
3944
4121
|
raise EOFError
|
3945
4122
|
|
3946
4123
|
sz = struct.unpack('<I', d)[0]
|
3947
|
-
d = self._input.read(sz)
|
4124
|
+
d = await self._input.read(sz)
|
3948
4125
|
if len(d) != sz:
|
3949
4126
|
raise EOFError
|
3950
4127
|
|
3951
4128
|
j = json.loads(d.decode('utf-8'))
|
3952
4129
|
return self._msh.unmarshal_obj(j, ty)
|
3953
4130
|
|
3954
|
-
def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
3955
|
-
with self.
|
3956
|
-
return self._recv_obj(ty)
|
4131
|
+
async def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
4132
|
+
async with self._input_lock:
|
4133
|
+
return await self._recv_obj(ty)
|
3957
4134
|
|
3958
4135
|
|
3959
4136
|
########################################
|
@@ -3987,7 +4164,7 @@ def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
|
|
3987
4164
|
return args
|
3988
4165
|
|
3989
4166
|
|
3990
|
-
def
|
4167
|
+
def prepare_subprocess_invocation(
|
3991
4168
|
*args: str,
|
3992
4169
|
env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
3993
4170
|
extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
@@ -3995,9 +4172,9 @@ def _prepare_subprocess_invocation(
|
|
3995
4172
|
shell: bool = False,
|
3996
4173
|
**kwargs: ta.Any,
|
3997
4174
|
) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
|
3998
|
-
log.debug(args)
|
4175
|
+
log.debug('prepare_subprocess_invocation: args=%r', args)
|
3999
4176
|
if extra_env:
|
4000
|
-
log.debug(extra_env)
|
4177
|
+
log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
|
4001
4178
|
|
4002
4179
|
if extra_env:
|
4003
4180
|
env = {**(env if env is not None else os.environ), **extra_env}
|
@@ -4016,14 +4193,46 @@ def _prepare_subprocess_invocation(
|
|
4016
4193
|
)
|
4017
4194
|
|
4018
4195
|
|
4019
|
-
|
4020
|
-
|
4021
|
-
|
4196
|
+
##
|
4197
|
+
|
4198
|
+
|
4199
|
+
@contextlib.contextmanager
|
4200
|
+
def subprocess_common_context(*args: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
|
4201
|
+
start_time = time.time()
|
4202
|
+
try:
|
4203
|
+
log.debug('subprocess_common_context.try: args=%r', args)
|
4204
|
+
yield
|
4205
|
+
|
4206
|
+
except Exception as exc: # noqa
|
4207
|
+
log.debug('subprocess_common_context.except: exc=%r', exc)
|
4208
|
+
raise
|
4209
|
+
|
4210
|
+
finally:
|
4211
|
+
end_time = time.time()
|
4212
|
+
elapsed_s = end_time - start_time
|
4213
|
+
log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
|
4214
|
+
|
4215
|
+
|
4216
|
+
##
|
4217
|
+
|
4218
|
+
|
4219
|
+
def subprocess_check_call(
|
4220
|
+
*args: str,
|
4221
|
+
stdout: ta.Any = sys.stderr,
|
4222
|
+
**kwargs: ta.Any,
|
4223
|
+
) -> None:
|
4224
|
+
args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
|
4225
|
+
with subprocess_common_context(*args, **kwargs):
|
4226
|
+
return subprocess.check_call(args, **kwargs) # type: ignore
|
4022
4227
|
|
4023
4228
|
|
4024
|
-
def subprocess_check_output(
|
4025
|
-
|
4026
|
-
|
4229
|
+
def subprocess_check_output(
|
4230
|
+
*args: str,
|
4231
|
+
**kwargs: ta.Any,
|
4232
|
+
) -> bytes:
|
4233
|
+
args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
|
4234
|
+
with subprocess_common_context(*args, **kwargs):
|
4235
|
+
return subprocess.check_output(args, **kwargs)
|
4027
4236
|
|
4028
4237
|
|
4029
4238
|
def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
|
@@ -4039,16 +4248,31 @@ DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
|
|
4039
4248
|
)
|
4040
4249
|
|
4041
4250
|
|
4042
|
-
def
|
4043
|
-
|
4251
|
+
def _subprocess_try_run(
|
4252
|
+
fn: ta.Callable[..., T],
|
4253
|
+
*args: ta.Any,
|
4044
4254
|
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4045
4255
|
**kwargs: ta.Any,
|
4046
|
-
) ->
|
4256
|
+
) -> ta.Union[T, Exception]:
|
4047
4257
|
try:
|
4048
|
-
|
4258
|
+
return fn(*args, **kwargs)
|
4049
4259
|
except try_exceptions as e: # noqa
|
4050
4260
|
if log.isEnabledFor(logging.DEBUG):
|
4051
4261
|
log.exception('command failed')
|
4262
|
+
return e
|
4263
|
+
|
4264
|
+
|
4265
|
+
def subprocess_try_call(
|
4266
|
+
*args: str,
|
4267
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4268
|
+
**kwargs: ta.Any,
|
4269
|
+
) -> bool:
|
4270
|
+
if isinstance(_subprocess_try_run(
|
4271
|
+
subprocess_check_call,
|
4272
|
+
*args,
|
4273
|
+
try_exceptions=try_exceptions,
|
4274
|
+
**kwargs,
|
4275
|
+
), Exception):
|
4052
4276
|
return False
|
4053
4277
|
else:
|
4054
4278
|
return True
|
@@ -4059,12 +4283,15 @@ def subprocess_try_output(
|
|
4059
4283
|
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4060
4284
|
**kwargs: ta.Any,
|
4061
4285
|
) -> ta.Optional[bytes]:
|
4062
|
-
|
4063
|
-
|
4064
|
-
|
4065
|
-
|
4066
|
-
|
4286
|
+
if isinstance(ret := _subprocess_try_run(
|
4287
|
+
subprocess_check_output,
|
4288
|
+
*args,
|
4289
|
+
try_exceptions=try_exceptions,
|
4290
|
+
**kwargs,
|
4291
|
+
), Exception):
|
4067
4292
|
return None
|
4293
|
+
else:
|
4294
|
+
return ret
|
4068
4295
|
|
4069
4296
|
|
4070
4297
|
def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|
@@ -4091,175 +4318,922 @@ def subprocess_close(
|
|
4091
4318
|
|
4092
4319
|
|
4093
4320
|
########################################
|
4094
|
-
#
|
4321
|
+
# ../remote/execution.py
|
4095
4322
|
|
4096
4323
|
|
4097
|
-
|
4098
|
-
class InterpInspection:
|
4099
|
-
exe: str
|
4100
|
-
version: Version
|
4324
|
+
##
|
4101
4325
|
|
4102
|
-
version_str: str
|
4103
|
-
config_vars: ta.Mapping[str, str]
|
4104
|
-
prefix: str
|
4105
|
-
base_prefix: str
|
4106
4326
|
|
4107
|
-
|
4108
|
-
|
4109
|
-
|
4110
|
-
|
4111
|
-
debug=bool(self.config_vars.get('Py_DEBUG')),
|
4112
|
-
)
|
4327
|
+
class _RemoteProtocol:
|
4328
|
+
class Message(abc.ABC): # noqa
|
4329
|
+
async def send(self, chan: RemoteChannel) -> None:
|
4330
|
+
await chan.send_obj(self, _RemoteProtocol.Message)
|
4113
4331
|
|
4114
|
-
|
4115
|
-
|
4116
|
-
|
4117
|
-
version=self.version,
|
4118
|
-
opts=self.opts,
|
4119
|
-
)
|
4332
|
+
@classmethod
|
4333
|
+
async def recv(cls: ta.Type[T], chan: RemoteChannel) -> ta.Optional[T]:
|
4334
|
+
return await chan.recv_obj(cls)
|
4120
4335
|
|
4121
|
-
|
4122
|
-
def is_venv(self) -> bool:
|
4123
|
-
return self.prefix != self.base_prefix
|
4336
|
+
#
|
4124
4337
|
|
4338
|
+
class Request(Message, abc.ABC): # noqa
|
4339
|
+
pass
|
4125
4340
|
|
4126
|
-
|
4341
|
+
@dc.dataclass(frozen=True)
|
4342
|
+
class CommandRequest(Request):
|
4343
|
+
seq: int
|
4344
|
+
cmd: Command
|
4127
4345
|
|
4128
|
-
|
4129
|
-
|
4346
|
+
@dc.dataclass(frozen=True)
|
4347
|
+
class PingRequest(Request):
|
4348
|
+
time: float
|
4130
4349
|
|
4131
|
-
|
4350
|
+
#
|
4132
4351
|
|
4133
|
-
|
4134
|
-
|
4135
|
-
version_str=__import__('sys').version,
|
4136
|
-
prefix=__import__('sys').prefix,
|
4137
|
-
base_prefix=__import__('sys').base_prefix,
|
4138
|
-
config_vars=__import__('sysconfig').get_config_vars(),
|
4139
|
-
))"""
|
4352
|
+
class Response(Message, abc.ABC): # noqa
|
4353
|
+
pass
|
4140
4354
|
|
4141
|
-
|
4355
|
+
@dc.dataclass(frozen=True)
|
4356
|
+
class LogResponse(Response):
|
4357
|
+
s: str
|
4142
4358
|
|
4143
|
-
@
|
4144
|
-
|
4145
|
-
|
4146
|
-
|
4147
|
-
) -> InterpInspection:
|
4148
|
-
dct = json.loads(output)
|
4359
|
+
@dc.dataclass(frozen=True)
|
4360
|
+
class CommandResponse(Response):
|
4361
|
+
seq: int
|
4362
|
+
res: CommandOutputOrExceptionData
|
4149
4363
|
|
4150
|
-
|
4364
|
+
@dc.dataclass(frozen=True)
|
4365
|
+
class PingResponse(Response):
|
4366
|
+
time: float
|
4151
4367
|
|
4152
|
-
return InterpInspection(
|
4153
|
-
exe=exe,
|
4154
|
-
version=version,
|
4155
|
-
**{k: dct[k] for k in (
|
4156
|
-
'version_str',
|
4157
|
-
'prefix',
|
4158
|
-
'base_prefix',
|
4159
|
-
'config_vars',
|
4160
|
-
)},
|
4161
|
-
)
|
4162
4368
|
|
4163
|
-
|
4164
|
-
def running(cls) -> 'InterpInspection':
|
4165
|
-
return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
|
4369
|
+
##
|
4166
4370
|
|
4167
|
-
def _inspect(self, exe: str) -> InterpInspection:
|
4168
|
-
output = subprocess_check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
|
4169
|
-
return self._build_inspection(exe, output.decode())
|
4170
4371
|
|
4171
|
-
|
4172
|
-
|
4173
|
-
|
4174
|
-
|
4175
|
-
|
4176
|
-
|
4177
|
-
|
4178
|
-
except Exception as e: # noqa
|
4179
|
-
if log.isEnabledFor(logging.DEBUG):
|
4180
|
-
log.exception('Failed to inspect interp: %s', exe)
|
4181
|
-
ret = None
|
4182
|
-
self._cache[exe] = ret
|
4183
|
-
return ret
|
4372
|
+
class _RemoteLogHandler(logging.Handler):
|
4373
|
+
def __init__(
|
4374
|
+
self,
|
4375
|
+
chan: RemoteChannel,
|
4376
|
+
loop: ta.Any = None,
|
4377
|
+
) -> None:
|
4378
|
+
super().__init__()
|
4184
4379
|
|
4380
|
+
self._chan = chan
|
4381
|
+
self._loop = loop
|
4185
4382
|
|
4186
|
-
|
4383
|
+
def emit(self, record):
|
4384
|
+
msg = self.format(record)
|
4187
4385
|
|
4386
|
+
async def inner():
|
4387
|
+
await _RemoteProtocol.LogResponse(msg).send(self._chan)
|
4188
4388
|
|
4189
|
-
|
4190
|
-
|
4389
|
+
loop = self._loop
|
4390
|
+
if loop is None:
|
4391
|
+
loop = asyncio.get_running_loop()
|
4392
|
+
if loop is not None:
|
4393
|
+
asyncio.run_coroutine_threadsafe(inner(), loop)
|
4191
4394
|
|
4192
4395
|
|
4193
4396
|
##
|
4194
4397
|
|
4195
4398
|
|
4196
|
-
|
4197
|
-
|
4198
|
-
|
4199
|
-
|
4200
|
-
|
4201
|
-
|
4202
|
-
|
4399
|
+
class _RemoteCommandHandler:
|
4400
|
+
def __init__(
|
4401
|
+
self,
|
4402
|
+
chan: RemoteChannel,
|
4403
|
+
executor: CommandExecutor,
|
4404
|
+
*,
|
4405
|
+
stop: ta.Optional[asyncio.Event] = None,
|
4406
|
+
) -> None:
|
4407
|
+
super().__init__()
|
4203
4408
|
|
4204
|
-
|
4205
|
-
|
4409
|
+
self._chan = chan
|
4410
|
+
self._executor = executor
|
4411
|
+
self._stop = stop if stop is not None else asyncio.Event()
|
4206
4412
|
|
4207
|
-
|
4208
|
-
|
4413
|
+
self._cmds_by_seq: ta.Dict[int, _RemoteCommandHandler._Command] = {}
|
4414
|
+
|
4415
|
+
@dc.dataclass(frozen=True)
|
4416
|
+
class _Command:
|
4417
|
+
req: _RemoteProtocol.CommandRequest
|
4418
|
+
fut: asyncio.Future
|
4419
|
+
|
4420
|
+
async def run(self) -> None:
|
4421
|
+
stop_task = asyncio.create_task(self._stop.wait())
|
4422
|
+
recv_task: ta.Optional[asyncio.Task] = None
|
4423
|
+
|
4424
|
+
while not self._stop.is_set():
|
4425
|
+
if recv_task is None:
|
4426
|
+
recv_task = asyncio.create_task(_RemoteProtocol.Request.recv(self._chan))
|
4427
|
+
|
4428
|
+
done, pending = await asyncio.wait([
|
4429
|
+
stop_task,
|
4430
|
+
recv_task,
|
4431
|
+
], return_when=asyncio.FIRST_COMPLETED)
|
4432
|
+
|
4433
|
+
if recv_task in done:
|
4434
|
+
msg: ta.Optional[_RemoteProtocol.Message] = check_isinstance(
|
4435
|
+
recv_task.result(),
|
4436
|
+
(_RemoteProtocol.Message, type(None)),
|
4437
|
+
)
|
4438
|
+
recv_task = None
|
4439
|
+
|
4440
|
+
if msg is None:
|
4441
|
+
break
|
4442
|
+
|
4443
|
+
await self._handle_message(msg)
|
4444
|
+
|
4445
|
+
async def _handle_message(self, msg: _RemoteProtocol.Message) -> None:
|
4446
|
+
if isinstance(msg, _RemoteProtocol.PingRequest):
|
4447
|
+
log.debug('Ping: %r', msg)
|
4448
|
+
await _RemoteProtocol.PingResponse(
|
4449
|
+
time=msg.time,
|
4450
|
+
).send(self._chan)
|
4451
|
+
|
4452
|
+
elif isinstance(msg, _RemoteProtocol.CommandRequest):
|
4453
|
+
fut = asyncio.create_task(self._handle_command_request(msg))
|
4454
|
+
self._cmds_by_seq[msg.seq] = _RemoteCommandHandler._Command(
|
4455
|
+
req=msg,
|
4456
|
+
fut=fut,
|
4457
|
+
)
|
4458
|
+
|
4459
|
+
else:
|
4460
|
+
raise TypeError(msg)
|
4461
|
+
|
4462
|
+
async def _handle_command_request(self, req: _RemoteProtocol.CommandRequest) -> None:
|
4463
|
+
res = await self._executor.try_execute(
|
4464
|
+
req.cmd,
|
4465
|
+
log=log,
|
4466
|
+
omit_exc_object=True,
|
4467
|
+
)
|
4468
|
+
|
4469
|
+
await _RemoteProtocol.CommandResponse(
|
4470
|
+
seq=req.seq,
|
4471
|
+
res=CommandOutputOrExceptionData(
|
4472
|
+
output=res.output,
|
4473
|
+
exception=res.exception,
|
4474
|
+
),
|
4475
|
+
).send(self._chan)
|
4476
|
+
|
4477
|
+
self._cmds_by_seq.pop(req.seq) # noqa
|
4478
|
+
|
4479
|
+
|
4480
|
+
##
|
4481
|
+
|
4482
|
+
|
4483
|
+
@dc.dataclass()
|
4484
|
+
class RemoteCommandError(Exception):
|
4485
|
+
e: CommandException
|
4486
|
+
|
4487
|
+
|
4488
|
+
class RemoteCommandExecutor(CommandExecutor):
|
4489
|
+
def __init__(self, chan: RemoteChannel) -> None:
|
4490
|
+
super().__init__()
|
4491
|
+
|
4492
|
+
self._chan = chan
|
4493
|
+
|
4494
|
+
self._cmd_seq = itertools.count()
|
4495
|
+
self._queue: asyncio.Queue = asyncio.Queue() # asyncio.Queue[RemoteCommandExecutor._Request]
|
4496
|
+
self._stop = asyncio.Event()
|
4497
|
+
self._loop_task: ta.Optional[asyncio.Task] = None
|
4498
|
+
self._reqs_by_seq: ta.Dict[int, RemoteCommandExecutor._Request] = {}
|
4499
|
+
|
4500
|
+
#
|
4501
|
+
|
4502
|
+
async def start(self) -> None:
|
4503
|
+
check_none(self._loop_task)
|
4504
|
+
check_state(not self._stop.is_set())
|
4505
|
+
self._loop_task = asyncio.create_task(self._loop())
|
4506
|
+
|
4507
|
+
async def aclose(self) -> None:
|
4508
|
+
self._stop.set()
|
4509
|
+
if self._loop_task is not None:
|
4510
|
+
await self._loop_task
|
4511
|
+
|
4512
|
+
#
|
4513
|
+
|
4514
|
+
@dc.dataclass(frozen=True)
|
4515
|
+
class _Request:
|
4516
|
+
seq: int
|
4517
|
+
cmd: Command
|
4518
|
+
fut: asyncio.Future
|
4519
|
+
|
4520
|
+
async def _loop(self) -> None:
|
4521
|
+
log.debug('RemoteCommandExecutor loop start: %r', self)
|
4522
|
+
|
4523
|
+
stop_task = asyncio.create_task(self._stop.wait())
|
4524
|
+
queue_task: ta.Optional[asyncio.Task] = None
|
4525
|
+
recv_task: ta.Optional[asyncio.Task] = None
|
4526
|
+
|
4527
|
+
while not self._stop.is_set():
|
4528
|
+
if queue_task is None:
|
4529
|
+
queue_task = asyncio.create_task(self._queue.get())
|
4530
|
+
if recv_task is None:
|
4531
|
+
recv_task = asyncio.create_task(_RemoteProtocol.Message.recv(self._chan))
|
4532
|
+
|
4533
|
+
done, pending = await asyncio.wait([
|
4534
|
+
stop_task,
|
4535
|
+
queue_task,
|
4536
|
+
recv_task,
|
4537
|
+
], return_when=asyncio.FIRST_COMPLETED)
|
4538
|
+
|
4539
|
+
if queue_task in done:
|
4540
|
+
req = check_isinstance(queue_task.result(), RemoteCommandExecutor._Request)
|
4541
|
+
queue_task = None
|
4542
|
+
await self._handle_request(req)
|
4543
|
+
|
4544
|
+
if recv_task in done:
|
4545
|
+
msg: ta.Optional[_RemoteProtocol.Message] = check_isinstance(
|
4546
|
+
recv_task.result(),
|
4547
|
+
(_RemoteProtocol.Message, type(None)),
|
4548
|
+
)
|
4549
|
+
recv_task = None
|
4550
|
+
|
4551
|
+
if msg is None:
|
4552
|
+
log.debug('RemoteCommandExecutor got eof: %r', self)
|
4553
|
+
break
|
4554
|
+
|
4555
|
+
await self._handle_message(msg)
|
4556
|
+
|
4557
|
+
log.debug('RemoteCommandExecutor loop stopping: %r', self)
|
4558
|
+
|
4559
|
+
for task in [
|
4560
|
+
stop_task,
|
4561
|
+
queue_task,
|
4562
|
+
recv_task,
|
4563
|
+
]:
|
4564
|
+
if task is not None and not task.done():
|
4565
|
+
task.cancel()
|
4566
|
+
|
4567
|
+
for req in self._reqs_by_seq.values():
|
4568
|
+
req.fut.cancel()
|
4569
|
+
|
4570
|
+
log.debug('RemoteCommandExecutor loop exited: %r', self)
|
4571
|
+
|
4572
|
+
async def _handle_request(self, req: _Request) -> None:
|
4573
|
+
self._reqs_by_seq[req.seq] = req
|
4574
|
+
await _RemoteProtocol.CommandRequest(
|
4575
|
+
seq=req.seq,
|
4576
|
+
cmd=req.cmd,
|
4577
|
+
).send(self._chan)
|
4578
|
+
|
4579
|
+
async def _handle_message(self, msg: _RemoteProtocol.Message) -> None:
|
4580
|
+
if isinstance(msg, _RemoteProtocol.PingRequest):
|
4581
|
+
log.debug('Ping: %r', msg)
|
4582
|
+
await _RemoteProtocol.PingResponse(
|
4583
|
+
time=msg.time,
|
4584
|
+
).send(self._chan)
|
4585
|
+
|
4586
|
+
elif isinstance(msg, _RemoteProtocol.LogResponse):
|
4587
|
+
log.info(msg.s)
|
4588
|
+
|
4589
|
+
elif isinstance(msg, _RemoteProtocol.CommandResponse):
|
4590
|
+
req = self._reqs_by_seq.pop(msg.seq)
|
4591
|
+
req.fut.set_result(msg.res)
|
4592
|
+
|
4593
|
+
else:
|
4594
|
+
raise TypeError(msg)
|
4595
|
+
|
4596
|
+
#
|
4597
|
+
|
4598
|
+
async def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
|
4599
|
+
req = RemoteCommandExecutor._Request(
|
4600
|
+
seq=next(self._cmd_seq),
|
4601
|
+
cmd=cmd,
|
4602
|
+
fut=asyncio.Future(),
|
4603
|
+
)
|
4604
|
+
await self._queue.put(req)
|
4605
|
+
return await req.fut
|
4606
|
+
|
4607
|
+
# @ta.override
|
4608
|
+
async def execute(self, cmd: Command) -> Command.Output:
|
4609
|
+
r = await self._remote_execute(cmd)
|
4610
|
+
if (e := r.exception) is not None:
|
4611
|
+
raise RemoteCommandError(e)
|
4612
|
+
else:
|
4613
|
+
return check_not_none(r.output)
|
4614
|
+
|
4615
|
+
# @ta.override
|
4616
|
+
async def try_execute(
|
4617
|
+
self,
|
4618
|
+
cmd: Command,
|
4619
|
+
*,
|
4620
|
+
log: ta.Optional[logging.Logger] = None,
|
4621
|
+
omit_exc_object: bool = False,
|
4622
|
+
) -> CommandOutputOrException:
|
4623
|
+
try:
|
4624
|
+
r = await self._remote_execute(cmd)
|
4625
|
+
|
4626
|
+
except Exception as e: # noqa
|
4627
|
+
if log is not None:
|
4628
|
+
log.exception('Exception executing remote command: %r', type(cmd))
|
4629
|
+
|
4630
|
+
return CommandOutputOrExceptionData(exception=CommandException.of(
|
4631
|
+
e,
|
4632
|
+
omit_exc_object=omit_exc_object,
|
4633
|
+
cmd=cmd,
|
4634
|
+
))
|
4635
|
+
|
4636
|
+
else:
|
4637
|
+
return r
|
4638
|
+
|
4639
|
+
|
4640
|
+
########################################
|
4641
|
+
# ../../../omlish/lite/asyncio/subprocesses.py
|
4642
|
+
|
4643
|
+
|
4644
|
+
##
|
4645
|
+
|
4646
|
+
|
4647
|
+
@contextlib.asynccontextmanager
|
4648
|
+
async def asyncio_subprocess_popen(
|
4649
|
+
*cmd: str,
|
4650
|
+
shell: bool = False,
|
4651
|
+
timeout: ta.Optional[float] = None,
|
4652
|
+
**kwargs: ta.Any,
|
4653
|
+
) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
|
4654
|
+
fac: ta.Any
|
4655
|
+
if shell:
|
4656
|
+
fac = functools.partial(
|
4657
|
+
asyncio.create_subprocess_shell,
|
4658
|
+
check_single(cmd),
|
4659
|
+
)
|
4660
|
+
else:
|
4661
|
+
fac = functools.partial(
|
4662
|
+
asyncio.create_subprocess_exec,
|
4663
|
+
*cmd,
|
4664
|
+
)
|
4665
|
+
|
4666
|
+
with subprocess_common_context(
|
4667
|
+
*cmd,
|
4668
|
+
shell=shell,
|
4669
|
+
timeout=timeout,
|
4670
|
+
**kwargs,
|
4671
|
+
):
|
4672
|
+
proc: asyncio.subprocess.Process
|
4673
|
+
proc = await fac(**kwargs)
|
4674
|
+
try:
|
4675
|
+
yield proc
|
4676
|
+
|
4677
|
+
finally:
|
4678
|
+
await asyncio_maybe_timeout(proc.wait(), timeout)
|
4679
|
+
|
4680
|
+
|
4681
|
+
##
|
4682
|
+
|
4683
|
+
|
4684
|
+
class AsyncioProcessCommunicator:
|
4685
|
+
def __init__(
|
4686
|
+
self,
|
4687
|
+
proc: asyncio.subprocess.Process,
|
4688
|
+
loop: ta.Optional[ta.Any] = None,
|
4689
|
+
) -> None:
|
4690
|
+
super().__init__()
|
4691
|
+
|
4692
|
+
if loop is None:
|
4693
|
+
loop = asyncio.get_running_loop()
|
4694
|
+
|
4695
|
+
self._proc = proc
|
4696
|
+
self._loop = loop
|
4697
|
+
|
4698
|
+
self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check_isinstance(
|
4699
|
+
proc._transport, # type: ignore # noqa
|
4700
|
+
asyncio.base_subprocess.BaseSubprocessTransport,
|
4701
|
+
)
|
4702
|
+
|
4703
|
+
@property
|
4704
|
+
def _debug(self) -> bool:
|
4705
|
+
return self._loop.get_debug()
|
4706
|
+
|
4707
|
+
async def _feed_stdin(self, input: bytes) -> None: # noqa
|
4708
|
+
stdin = check_not_none(self._proc.stdin)
|
4709
|
+
try:
|
4710
|
+
if input is not None:
|
4711
|
+
stdin.write(input)
|
4712
|
+
if self._debug:
|
4713
|
+
log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
|
4714
|
+
|
4715
|
+
await stdin.drain()
|
4716
|
+
|
4717
|
+
except (BrokenPipeError, ConnectionResetError) as exc:
|
4718
|
+
# communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
|
4719
|
+
# exceptions.
|
4720
|
+
if self._debug:
|
4721
|
+
log.debug('%r communicate: stdin got %r', self, exc)
|
4722
|
+
|
4723
|
+
if self._debug:
|
4724
|
+
log.debug('%r communicate: close stdin', self)
|
4725
|
+
|
4726
|
+
stdin.close()
|
4727
|
+
|
4728
|
+
async def _noop(self) -> None:
|
4729
|
+
return None
|
4730
|
+
|
4731
|
+
async def _read_stream(self, fd: int) -> bytes:
|
4732
|
+
transport: ta.Any = check_not_none(self._transport.get_pipe_transport(fd))
|
4733
|
+
|
4734
|
+
if fd == 2:
|
4735
|
+
stream = check_not_none(self._proc.stderr)
|
4736
|
+
else:
|
4737
|
+
check_equal(fd, 1)
|
4738
|
+
stream = check_not_none(self._proc.stdout)
|
4739
|
+
|
4740
|
+
if self._debug:
|
4741
|
+
name = 'stdout' if fd == 1 else 'stderr'
|
4742
|
+
log.debug('%r communicate: read %s', self, name)
|
4743
|
+
|
4744
|
+
output = await stream.read()
|
4745
|
+
|
4746
|
+
if self._debug:
|
4747
|
+
name = 'stdout' if fd == 1 else 'stderr'
|
4748
|
+
log.debug('%r communicate: close %s', self, name)
|
4749
|
+
|
4750
|
+
transport.close()
|
4751
|
+
|
4752
|
+
return output
|
4753
|
+
|
4754
|
+
class Communication(ta.NamedTuple):
|
4755
|
+
stdout: ta.Optional[bytes]
|
4756
|
+
stderr: ta.Optional[bytes]
|
4757
|
+
|
4758
|
+
async def _communicate(
|
4759
|
+
self,
|
4760
|
+
input: ta.Any = None, # noqa
|
4761
|
+
) -> Communication:
|
4762
|
+
stdin_fut: ta.Any
|
4763
|
+
if self._proc.stdin is not None:
|
4764
|
+
stdin_fut = self._feed_stdin(input)
|
4765
|
+
else:
|
4766
|
+
stdin_fut = self._noop()
|
4767
|
+
|
4768
|
+
stdout_fut: ta.Any
|
4769
|
+
if self._proc.stdout is not None:
|
4770
|
+
stdout_fut = self._read_stream(1)
|
4771
|
+
else:
|
4772
|
+
stdout_fut = self._noop()
|
4773
|
+
|
4774
|
+
stderr_fut: ta.Any
|
4775
|
+
if self._proc.stderr is not None:
|
4776
|
+
stderr_fut = self._read_stream(2)
|
4777
|
+
else:
|
4778
|
+
stderr_fut = self._noop()
|
4779
|
+
|
4780
|
+
stdin_res, stdout_res, stderr_res = await asyncio.gather(stdin_fut, stdout_fut, stderr_fut)
|
4781
|
+
|
4782
|
+
await self._proc.wait()
|
4783
|
+
|
4784
|
+
return AsyncioProcessCommunicator.Communication(stdout_res, stderr_res)
|
4785
|
+
|
4786
|
+
async def communicate(
|
4787
|
+
self,
|
4788
|
+
input: ta.Any = None, # noqa
|
4789
|
+
timeout: ta.Optional[float] = None,
|
4790
|
+
) -> Communication:
|
4791
|
+
return await asyncio_maybe_timeout(self._communicate(input), timeout)
|
4792
|
+
|
4793
|
+
|
4794
|
+
async def asyncio_subprocess_communicate(
|
4795
|
+
proc: asyncio.subprocess.Process,
|
4796
|
+
input: ta.Any = None, # noqa
|
4797
|
+
timeout: ta.Optional[float] = None,
|
4798
|
+
) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
|
4799
|
+
return await AsyncioProcessCommunicator(proc).communicate(input, timeout) # noqa
|
4800
|
+
|
4801
|
+
|
4802
|
+
##
|
4803
|
+
|
4804
|
+
|
4805
|
+
async def _asyncio_subprocess_check_run(
|
4806
|
+
*args: str,
|
4807
|
+
input: ta.Any = None, # noqa
|
4808
|
+
timeout: ta.Optional[float] = None,
|
4809
|
+
**kwargs: ta.Any,
|
4810
|
+
) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
|
4811
|
+
args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
|
4812
|
+
|
4813
|
+
proc: asyncio.subprocess.Process
|
4814
|
+
async with asyncio_subprocess_popen(*args, **kwargs) as proc:
|
4815
|
+
stdout, stderr = await asyncio_subprocess_communicate(proc, input, timeout)
|
4816
|
+
|
4817
|
+
if proc.returncode:
|
4818
|
+
raise subprocess.CalledProcessError(
|
4819
|
+
proc.returncode,
|
4820
|
+
args,
|
4821
|
+
output=stdout,
|
4822
|
+
stderr=stderr,
|
4823
|
+
)
|
4824
|
+
|
4825
|
+
return stdout, stderr
|
4826
|
+
|
4827
|
+
|
4828
|
+
async def asyncio_subprocess_check_call(
|
4829
|
+
*args: str,
|
4830
|
+
stdout: ta.Any = sys.stderr,
|
4831
|
+
input: ta.Any = None, # noqa
|
4832
|
+
timeout: ta.Optional[float] = None,
|
4833
|
+
**kwargs: ta.Any,
|
4834
|
+
) -> None:
|
4835
|
+
_, _ = await _asyncio_subprocess_check_run(
|
4836
|
+
*args,
|
4837
|
+
stdout=stdout,
|
4838
|
+
input=input,
|
4839
|
+
timeout=timeout,
|
4840
|
+
**kwargs,
|
4841
|
+
)
|
4842
|
+
|
4843
|
+
|
4844
|
+
async def asyncio_subprocess_check_output(
|
4845
|
+
*args: str,
|
4846
|
+
input: ta.Any = None, # noqa
|
4847
|
+
timeout: ta.Optional[float] = None,
|
4848
|
+
**kwargs: ta.Any,
|
4849
|
+
) -> bytes:
|
4850
|
+
stdout, stderr = await _asyncio_subprocess_check_run(
|
4851
|
+
*args,
|
4852
|
+
stdout=asyncio.subprocess.PIPE,
|
4853
|
+
input=input,
|
4854
|
+
timeout=timeout,
|
4855
|
+
**kwargs,
|
4856
|
+
)
|
4857
|
+
|
4858
|
+
return check_not_none(stdout)
|
4859
|
+
|
4860
|
+
|
4861
|
+
async def asyncio_subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
|
4862
|
+
return (await asyncio_subprocess_check_output(*args, **kwargs)).decode().strip()
|
4863
|
+
|
4864
|
+
|
4865
|
+
##
|
4866
|
+
|
4867
|
+
|
4868
|
+
async def _asyncio_subprocess_try_run(
|
4869
|
+
fn: ta.Callable[..., ta.Awaitable[T]],
|
4870
|
+
*args: ta.Any,
|
4871
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4872
|
+
**kwargs: ta.Any,
|
4873
|
+
) -> ta.Union[T, Exception]:
|
4874
|
+
try:
|
4875
|
+
return await fn(*args, **kwargs)
|
4876
|
+
except try_exceptions as e: # noqa
|
4877
|
+
if log.isEnabledFor(logging.DEBUG):
|
4878
|
+
log.exception('command failed')
|
4879
|
+
return e
|
4880
|
+
|
4881
|
+
|
4882
|
+
async def asyncio_subprocess_try_call(
|
4883
|
+
*args: str,
|
4884
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4885
|
+
**kwargs: ta.Any,
|
4886
|
+
) -> bool:
|
4887
|
+
if isinstance(await _asyncio_subprocess_try_run(
|
4888
|
+
asyncio_subprocess_check_call,
|
4889
|
+
*args,
|
4890
|
+
try_exceptions=try_exceptions,
|
4891
|
+
**kwargs,
|
4892
|
+
), Exception):
|
4893
|
+
return False
|
4894
|
+
else:
|
4895
|
+
return True
|
4896
|
+
|
4897
|
+
|
4898
|
+
async def asyncio_subprocess_try_output(
|
4899
|
+
*args: str,
|
4900
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4901
|
+
**kwargs: ta.Any,
|
4902
|
+
) -> ta.Optional[bytes]:
|
4903
|
+
if isinstance(ret := await _asyncio_subprocess_try_run(
|
4904
|
+
asyncio_subprocess_check_output,
|
4905
|
+
*args,
|
4906
|
+
try_exceptions=try_exceptions,
|
4907
|
+
**kwargs,
|
4908
|
+
), Exception):
|
4909
|
+
return None
|
4910
|
+
else:
|
4911
|
+
return ret
|
4912
|
+
|
4913
|
+
|
4914
|
+
async def asyncio_subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|
4915
|
+
out = await asyncio_subprocess_try_output(*args, **kwargs)
|
4916
|
+
return out.decode().strip() if out is not None else None
|
4917
|
+
|
4918
|
+
|
4919
|
+
########################################
|
4920
|
+
# ../../../omdev/interp/inspect.py
|
4921
|
+
|
4922
|
+
|
4923
|
+
@dc.dataclass(frozen=True)
|
4924
|
+
class InterpInspection:
|
4925
|
+
exe: str
|
4926
|
+
version: Version
|
4927
|
+
|
4928
|
+
version_str: str
|
4929
|
+
config_vars: ta.Mapping[str, str]
|
4930
|
+
prefix: str
|
4931
|
+
base_prefix: str
|
4932
|
+
|
4933
|
+
@property
|
4934
|
+
def opts(self) -> InterpOpts:
|
4935
|
+
return InterpOpts(
|
4936
|
+
threaded=bool(self.config_vars.get('Py_GIL_DISABLED')),
|
4937
|
+
debug=bool(self.config_vars.get('Py_DEBUG')),
|
4938
|
+
)
|
4939
|
+
|
4940
|
+
@property
|
4941
|
+
def iv(self) -> InterpVersion:
|
4942
|
+
return InterpVersion(
|
4943
|
+
version=self.version,
|
4944
|
+
opts=self.opts,
|
4945
|
+
)
|
4946
|
+
|
4947
|
+
@property
|
4948
|
+
def is_venv(self) -> bool:
|
4949
|
+
return self.prefix != self.base_prefix
|
4950
|
+
|
4951
|
+
|
4952
|
+
class InterpInspector:
|
4953
|
+
def __init__(self) -> None:
|
4954
|
+
super().__init__()
|
4955
|
+
|
4956
|
+
self._cache: ta.Dict[str, ta.Optional[InterpInspection]] = {}
|
4957
|
+
|
4958
|
+
_RAW_INSPECTION_CODE = """
|
4959
|
+
__import__('json').dumps(dict(
|
4960
|
+
version_str=__import__('sys').version,
|
4961
|
+
prefix=__import__('sys').prefix,
|
4962
|
+
base_prefix=__import__('sys').base_prefix,
|
4963
|
+
config_vars=__import__('sysconfig').get_config_vars(),
|
4964
|
+
))"""
|
4965
|
+
|
4966
|
+
_INSPECTION_CODE = ''.join(l.strip() for l in _RAW_INSPECTION_CODE.splitlines())
|
4967
|
+
|
4968
|
+
@staticmethod
|
4969
|
+
def _build_inspection(
|
4970
|
+
exe: str,
|
4971
|
+
output: str,
|
4972
|
+
) -> InterpInspection:
|
4973
|
+
dct = json.loads(output)
|
4974
|
+
|
4975
|
+
version = Version(dct['version_str'].split()[0])
|
4976
|
+
|
4977
|
+
return InterpInspection(
|
4978
|
+
exe=exe,
|
4979
|
+
version=version,
|
4980
|
+
**{k: dct[k] for k in (
|
4981
|
+
'version_str',
|
4982
|
+
'prefix',
|
4983
|
+
'base_prefix',
|
4984
|
+
'config_vars',
|
4985
|
+
)},
|
4986
|
+
)
|
4987
|
+
|
4988
|
+
@classmethod
|
4989
|
+
def running(cls) -> 'InterpInspection':
|
4990
|
+
return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
|
4991
|
+
|
4992
|
+
async def _inspect(self, exe: str) -> InterpInspection:
|
4993
|
+
output = await asyncio_subprocess_check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
|
4994
|
+
return self._build_inspection(exe, output.decode())
|
4995
|
+
|
4996
|
+
async def inspect(self, exe: str) -> ta.Optional[InterpInspection]:
|
4997
|
+
try:
|
4998
|
+
return self._cache[exe]
|
4999
|
+
except KeyError:
|
5000
|
+
ret: ta.Optional[InterpInspection]
|
5001
|
+
try:
|
5002
|
+
ret = await self._inspect(exe)
|
5003
|
+
except Exception as e: # noqa
|
5004
|
+
if log.isEnabledFor(logging.DEBUG):
|
5005
|
+
log.exception('Failed to inspect interp: %s', exe)
|
5006
|
+
ret = None
|
5007
|
+
self._cache[exe] = ret
|
5008
|
+
return ret
|
5009
|
+
|
5010
|
+
|
5011
|
+
INTERP_INSPECTOR = InterpInspector()
|
5012
|
+
|
5013
|
+
|
5014
|
+
########################################
|
5015
|
+
# ../commands/subprocess.py
|
5016
|
+
|
5017
|
+
|
5018
|
+
##
|
5019
|
+
|
5020
|
+
|
5021
|
+
@dc.dataclass(frozen=True)
|
5022
|
+
class SubprocessCommand(Command['SubprocessCommand.Output']):
|
5023
|
+
cmd: ta.Sequence[str]
|
5024
|
+
|
5025
|
+
shell: bool = False
|
5026
|
+
cwd: ta.Optional[str] = None
|
5027
|
+
env: ta.Optional[ta.Mapping[str, str]] = None
|
5028
|
+
|
5029
|
+
stdout: str = 'pipe' # SubprocessChannelOption
|
5030
|
+
stderr: str = 'pipe' # SubprocessChannelOption
|
5031
|
+
|
5032
|
+
input: ta.Optional[bytes] = None
|
5033
|
+
timeout: ta.Optional[float] = None
|
4209
5034
|
|
4210
5035
|
def __post_init__(self) -> None:
|
4211
5036
|
check_not_isinstance(self.cmd, str)
|
4212
5037
|
|
4213
|
-
@dc.dataclass(frozen=True)
|
4214
|
-
class Output(Command.Output):
|
4215
|
-
rc: int
|
4216
|
-
pid: int
|
5038
|
+
@dc.dataclass(frozen=True)
|
5039
|
+
class Output(Command.Output):
|
5040
|
+
rc: int
|
5041
|
+
pid: int
|
5042
|
+
|
5043
|
+
elapsed_s: float
|
5044
|
+
|
5045
|
+
stdout: ta.Optional[bytes] = None
|
5046
|
+
stderr: ta.Optional[bytes] = None
|
5047
|
+
|
5048
|
+
|
5049
|
+
##
|
5050
|
+
|
5051
|
+
|
5052
|
+
class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
|
5053
|
+
async def execute(self, cmd: SubprocessCommand) -> SubprocessCommand.Output:
|
5054
|
+
proc: asyncio.subprocess.Process
|
5055
|
+
async with asyncio_subprocess_popen(
|
5056
|
+
*subprocess_maybe_shell_wrap_exec(*cmd.cmd),
|
5057
|
+
|
5058
|
+
shell=cmd.shell,
|
5059
|
+
cwd=cmd.cwd,
|
5060
|
+
env={**os.environ, **(cmd.env or {})},
|
5061
|
+
|
5062
|
+
stdin=subprocess.PIPE if cmd.input is not None else None,
|
5063
|
+
stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, cmd.stdout)],
|
5064
|
+
stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, cmd.stderr)],
|
5065
|
+
|
5066
|
+
timeout=cmd.timeout,
|
5067
|
+
) as proc:
|
5068
|
+
start_time = time.time()
|
5069
|
+
stdout, stderr = await asyncio_subprocess_communicate(
|
5070
|
+
proc,
|
5071
|
+
input=cmd.input,
|
5072
|
+
timeout=cmd.timeout,
|
5073
|
+
)
|
5074
|
+
end_time = time.time()
|
5075
|
+
|
5076
|
+
return SubprocessCommand.Output(
|
5077
|
+
rc=check_not_none(proc.returncode),
|
5078
|
+
pid=proc.pid,
|
5079
|
+
|
5080
|
+
elapsed_s=end_time - start_time,
|
5081
|
+
|
5082
|
+
stdout=stdout, # noqa
|
5083
|
+
stderr=stderr, # noqa
|
5084
|
+
)
|
5085
|
+
|
5086
|
+
|
5087
|
+
########################################
|
5088
|
+
# ../remote/_main.py
|
5089
|
+
|
5090
|
+
|
5091
|
+
##
|
5092
|
+
|
5093
|
+
|
5094
|
+
class _RemoteExecutionLogHandler(logging.Handler):
|
5095
|
+
def __init__(self, fn: ta.Callable[[str], None]) -> None:
|
5096
|
+
super().__init__()
|
5097
|
+
self._fn = fn
|
5098
|
+
|
5099
|
+
def emit(self, record):
|
5100
|
+
msg = self.format(record)
|
5101
|
+
self._fn(msg)
|
5102
|
+
|
5103
|
+
|
5104
|
+
##
|
5105
|
+
|
5106
|
+
|
5107
|
+
class _RemoteExecutionMain:
|
5108
|
+
def __init__(
|
5109
|
+
self,
|
5110
|
+
chan: RemoteChannel,
|
5111
|
+
) -> None:
|
5112
|
+
super().__init__()
|
5113
|
+
|
5114
|
+
self._chan = chan
|
5115
|
+
|
5116
|
+
self.__bootstrap: ta.Optional[MainBootstrap] = None
|
5117
|
+
self.__injector: ta.Optional[Injector] = None
|
5118
|
+
|
5119
|
+
@property
|
5120
|
+
def _bootstrap(self) -> MainBootstrap:
|
5121
|
+
return check_not_none(self.__bootstrap)
|
5122
|
+
|
5123
|
+
@property
|
5124
|
+
def _injector(self) -> Injector:
|
5125
|
+
return check_not_none(self.__injector)
|
5126
|
+
|
5127
|
+
#
|
5128
|
+
|
5129
|
+
def _timebomb_main(
|
5130
|
+
self,
|
5131
|
+
delay_s: float,
|
5132
|
+
*,
|
5133
|
+
sig: int = signal.SIGINT,
|
5134
|
+
code: int = 1,
|
5135
|
+
) -> None:
|
5136
|
+
time.sleep(delay_s)
|
5137
|
+
|
5138
|
+
if (pgid := os.getpgid(0)) == os.getpid():
|
5139
|
+
os.killpg(pgid, sig)
|
5140
|
+
|
5141
|
+
os._exit(code) # noqa
|
5142
|
+
|
5143
|
+
@cached_nullary
|
5144
|
+
def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
|
5145
|
+
if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
|
5146
|
+
return None
|
5147
|
+
|
5148
|
+
thr = threading.Thread(
|
5149
|
+
target=functools.partial(self._timebomb_main, tbd),
|
5150
|
+
name=f'{self.__class__.__name__}.timebomb',
|
5151
|
+
daemon=True,
|
5152
|
+
)
|
5153
|
+
|
5154
|
+
thr.start()
|
4217
5155
|
|
4218
|
-
|
5156
|
+
log.debug('Started timebomb thread: %r', thr)
|
4219
5157
|
|
4220
|
-
|
4221
|
-
stderr: ta.Optional[bytes] = None
|
5158
|
+
return thr
|
4222
5159
|
|
5160
|
+
#
|
4223
5161
|
|
4224
|
-
|
5162
|
+
@cached_nullary
|
5163
|
+
def _log_handler(self) -> _RemoteLogHandler:
|
5164
|
+
return _RemoteLogHandler(self._chan)
|
4225
5165
|
|
5166
|
+
#
|
4226
5167
|
|
4227
|
-
|
4228
|
-
|
4229
|
-
|
4230
|
-
subprocess_maybe_shell_wrap_exec(*inp.cmd),
|
5168
|
+
async def _setup(self) -> None:
|
5169
|
+
check_none(self.__bootstrap)
|
5170
|
+
check_none(self.__injector)
|
4231
5171
|
|
4232
|
-
|
4233
|
-
cwd=inp.cwd,
|
4234
|
-
env={**os.environ, **(inp.env or {})},
|
5172
|
+
# Bootstrap
|
4235
5173
|
|
4236
|
-
|
4237
|
-
stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stdout)],
|
4238
|
-
stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stderr)],
|
4239
|
-
) as proc:
|
4240
|
-
start_time = time.time()
|
4241
|
-
stdout, stderr = proc.communicate(
|
4242
|
-
input=inp.input,
|
4243
|
-
timeout=inp.timeout,
|
4244
|
-
)
|
4245
|
-
end_time = time.time()
|
5174
|
+
self.__bootstrap = check_not_none(await self._chan.recv_obj(MainBootstrap))
|
4246
5175
|
|
4247
|
-
|
4248
|
-
|
4249
|
-
pid=proc.pid,
|
5176
|
+
if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
|
5177
|
+
pycharm_debug_connect(prd)
|
4250
5178
|
|
4251
|
-
|
5179
|
+
self.__injector = main_bootstrap(self._bootstrap)
|
4252
5180
|
|
4253
|
-
|
4254
|
-
|
5181
|
+
self._chan.set_marshaler(self._injector[ObjMarshalerManager])
|
5182
|
+
|
5183
|
+
# Post-bootstrap
|
5184
|
+
|
5185
|
+
if self._bootstrap.remote_config.set_pgid:
|
5186
|
+
if os.getpgid(0) != os.getpid():
|
5187
|
+
log.debug('Setting pgid')
|
5188
|
+
os.setpgid(0, 0)
|
5189
|
+
|
5190
|
+
if (ds := self._bootstrap.remote_config.deathsig) is not None:
|
5191
|
+
log.debug('Setting deathsig: %s', ds)
|
5192
|
+
set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
|
5193
|
+
|
5194
|
+
self._timebomb_thread()
|
5195
|
+
|
5196
|
+
if self._bootstrap.remote_config.forward_logging:
|
5197
|
+
log.debug('Installing log forwarder')
|
5198
|
+
logging.root.addHandler(self._log_handler())
|
5199
|
+
|
5200
|
+
#
|
5201
|
+
|
5202
|
+
async def run(self) -> None:
|
5203
|
+
await self._setup()
|
5204
|
+
|
5205
|
+
executor = self._injector[LocalCommandExecutor]
|
5206
|
+
|
5207
|
+
handler = _RemoteCommandHandler(self._chan, executor)
|
5208
|
+
|
5209
|
+
await handler.run()
|
5210
|
+
|
5211
|
+
|
5212
|
+
def _remote_execution_main() -> None:
|
5213
|
+
rt = pyremote_bootstrap_finalize() # noqa
|
5214
|
+
|
5215
|
+
async def inner() -> None:
|
5216
|
+
input = await asyncio_open_stream_reader(rt.input) # noqa
|
5217
|
+
output = await asyncio_open_stream_writer(rt.output)
|
5218
|
+
|
5219
|
+
chan = RemoteChannelImpl(
|
5220
|
+
input,
|
5221
|
+
output,
|
4255
5222
|
)
|
4256
5223
|
|
5224
|
+
await _RemoteExecutionMain(chan).run()
|
5225
|
+
|
5226
|
+
asyncio.run(inner())
|
5227
|
+
|
4257
5228
|
|
4258
5229
|
########################################
|
4259
5230
|
# ../remote/spawning.py
|
4260
5231
|
|
4261
5232
|
|
4262
|
-
|
5233
|
+
##
|
5234
|
+
|
5235
|
+
|
5236
|
+
class RemoteSpawning(abc.ABC):
|
4263
5237
|
@dc.dataclass(frozen=True)
|
4264
5238
|
class Target:
|
4265
5239
|
shell: ta.Optional[str] = None
|
@@ -4270,15 +5244,35 @@ class RemoteSpawning:
|
|
4270
5244
|
|
4271
5245
|
stderr: ta.Optional[str] = None # SubprocessChannelOption
|
4272
5246
|
|
4273
|
-
|
5247
|
+
@dc.dataclass(frozen=True)
|
5248
|
+
class Spawned:
|
5249
|
+
stdin: asyncio.StreamWriter
|
5250
|
+
stdout: asyncio.StreamReader
|
5251
|
+
stderr: ta.Optional[asyncio.StreamReader]
|
5252
|
+
|
5253
|
+
@abc.abstractmethod
|
5254
|
+
def spawn(
|
5255
|
+
self,
|
5256
|
+
tgt: Target,
|
5257
|
+
src: str,
|
5258
|
+
*,
|
5259
|
+
timeout: ta.Optional[float] = None,
|
5260
|
+
debug: bool = False,
|
5261
|
+
) -> ta.AsyncContextManager[Spawned]:
|
5262
|
+
raise NotImplementedError
|
4274
5263
|
|
4275
|
-
|
5264
|
+
|
5265
|
+
##
|
5266
|
+
|
5267
|
+
|
5268
|
+
class SubprocessRemoteSpawning(RemoteSpawning):
|
5269
|
+
class _PreparedCmd(ta.NamedTuple): # noqa
|
4276
5270
|
cmd: ta.Sequence[str]
|
4277
5271
|
shell: bool
|
4278
5272
|
|
4279
5273
|
def _prepare_cmd(
|
4280
5274
|
self,
|
4281
|
-
tgt: Target,
|
5275
|
+
tgt: RemoteSpawning.Target,
|
4282
5276
|
src: str,
|
4283
5277
|
) -> _PreparedCmd:
|
4284
5278
|
if tgt.shell is not None:
|
@@ -4286,44 +5280,38 @@ class RemoteSpawning:
|
|
4286
5280
|
if tgt.shell_quote:
|
4287
5281
|
sh_src = shlex.quote(sh_src)
|
4288
5282
|
sh_cmd = f'{tgt.shell} {sh_src}'
|
4289
|
-
return
|
4290
|
-
cmd=[sh_cmd],
|
4291
|
-
shell=True,
|
4292
|
-
)
|
5283
|
+
return SubprocessRemoteSpawning._PreparedCmd([sh_cmd], shell=True)
|
4293
5284
|
|
4294
5285
|
else:
|
4295
|
-
return
|
4296
|
-
cmd=[tgt.python, '-c', src],
|
4297
|
-
shell=False,
|
4298
|
-
)
|
5286
|
+
return SubprocessRemoteSpawning._PreparedCmd([tgt.python, '-c', src], shell=False)
|
4299
5287
|
|
4300
5288
|
#
|
4301
5289
|
|
4302
|
-
@
|
4303
|
-
|
4304
|
-
stdin: ta.IO
|
4305
|
-
stdout: ta.IO
|
4306
|
-
stderr: ta.Optional[ta.IO]
|
4307
|
-
|
4308
|
-
@contextlib.contextmanager
|
4309
|
-
def spawn(
|
5290
|
+
@contextlib.asynccontextmanager
|
5291
|
+
async def spawn(
|
4310
5292
|
self,
|
4311
|
-
tgt: Target,
|
5293
|
+
tgt: RemoteSpawning.Target,
|
4312
5294
|
src: str,
|
4313
5295
|
*,
|
4314
5296
|
timeout: ta.Optional[float] = None,
|
4315
|
-
|
5297
|
+
debug: bool = False,
|
5298
|
+
) -> ta.AsyncGenerator[RemoteSpawning.Spawned, None]:
|
4316
5299
|
pc = self._prepare_cmd(tgt, src)
|
4317
5300
|
|
4318
|
-
|
4319
|
-
|
4320
|
-
|
4321
|
-
|
4322
|
-
|
4323
|
-
|
4324
|
-
|
4325
|
-
|
4326
|
-
|
5301
|
+
cmd = pc.cmd
|
5302
|
+
if not debug:
|
5303
|
+
cmd = subprocess_maybe_shell_wrap_exec(*cmd)
|
5304
|
+
|
5305
|
+
async with asyncio_subprocess_popen(
|
5306
|
+
*cmd,
|
5307
|
+
shell=pc.shell,
|
5308
|
+
stdin=subprocess.PIPE,
|
5309
|
+
stdout=subprocess.PIPE,
|
5310
|
+
stderr=(
|
5311
|
+
SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, tgt.stderr)]
|
5312
|
+
if tgt.stderr is not None else None
|
5313
|
+
),
|
5314
|
+
timeout=timeout,
|
4327
5315
|
) as proc:
|
4328
5316
|
stdin = check_not_none(proc.stdin)
|
4329
5317
|
stdout = check_not_none(proc.stdout)
|
@@ -4341,8 +5329,6 @@ class RemoteSpawning:
|
|
4341
5329
|
except BrokenPipeError:
|
4342
5330
|
pass
|
4343
5331
|
|
4344
|
-
proc.wait(timeout)
|
4345
|
-
|
4346
5332
|
|
4347
5333
|
########################################
|
4348
5334
|
# ../../../omdev/interp/providers.py
|
@@ -4371,17 +5357,17 @@ class InterpProvider(abc.ABC):
|
|
4371
5357
|
setattr(cls, 'name', snake_case(cls.__name__[:-len(sfx)]))
|
4372
5358
|
|
4373
5359
|
@abc.abstractmethod
|
4374
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5360
|
+
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Awaitable[ta.Sequence[InterpVersion]]:
|
4375
5361
|
raise NotImplementedError
|
4376
5362
|
|
4377
5363
|
@abc.abstractmethod
|
4378
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5364
|
+
def get_installed_version(self, version: InterpVersion) -> ta.Awaitable[Interp]:
|
4379
5365
|
raise NotImplementedError
|
4380
5366
|
|
4381
|
-
def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5367
|
+
async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
4382
5368
|
return []
|
4383
5369
|
|
4384
|
-
def install_version(self, version: InterpVersion) -> Interp:
|
5370
|
+
async def install_version(self, version: InterpVersion) -> Interp:
|
4385
5371
|
raise TypeError
|
4386
5372
|
|
4387
5373
|
|
@@ -4393,10 +5379,10 @@ class RunningInterpProvider(InterpProvider):
|
|
4393
5379
|
def version(self) -> InterpVersion:
|
4394
5380
|
return InterpInspector.running().iv
|
4395
5381
|
|
4396
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5382
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
4397
5383
|
return [self.version()]
|
4398
5384
|
|
4399
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5385
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
4400
5386
|
if version != self.version():
|
4401
5387
|
raise KeyError(version)
|
4402
5388
|
return Interp(
|
@@ -4406,159 +5392,26 @@ class RunningInterpProvider(InterpProvider):
|
|
4406
5392
|
|
4407
5393
|
|
4408
5394
|
########################################
|
4409
|
-
# ../remote/
|
4410
|
-
|
4411
|
-
|
4412
|
-
##
|
4413
|
-
|
4414
|
-
|
4415
|
-
class _RemoteExecutionLogHandler(logging.Handler):
|
4416
|
-
def __init__(self, fn: ta.Callable[[str], None]) -> None:
|
4417
|
-
super().__init__()
|
4418
|
-
self._fn = fn
|
4419
|
-
|
4420
|
-
def emit(self, record):
|
4421
|
-
msg = self.format(record)
|
4422
|
-
self._fn(msg)
|
4423
|
-
|
4424
|
-
|
4425
|
-
@dc.dataclass(frozen=True)
|
4426
|
-
class _RemoteExecutionRequest:
|
4427
|
-
c: Command
|
4428
|
-
|
4429
|
-
|
4430
|
-
@dc.dataclass(frozen=True)
|
4431
|
-
class _RemoteExecutionLog:
|
4432
|
-
s: str
|
4433
|
-
|
4434
|
-
|
4435
|
-
@dc.dataclass(frozen=True)
|
4436
|
-
class _RemoteExecutionResponse:
|
4437
|
-
r: ta.Optional[CommandOutputOrExceptionData] = None
|
4438
|
-
l: ta.Optional[_RemoteExecutionLog] = None
|
4439
|
-
|
4440
|
-
|
4441
|
-
def _remote_execution_main() -> None:
|
4442
|
-
rt = pyremote_bootstrap_finalize() # noqa
|
4443
|
-
|
4444
|
-
chan = RemoteChannel(
|
4445
|
-
rt.input,
|
4446
|
-
rt.output,
|
4447
|
-
)
|
4448
|
-
|
4449
|
-
bs = check_not_none(chan.recv_obj(MainBootstrap))
|
4450
|
-
|
4451
|
-
if (prd := bs.remote_config.pycharm_remote_debug) is not None:
|
4452
|
-
pycharm_debug_connect(prd)
|
4453
|
-
|
4454
|
-
injector = main_bootstrap(bs)
|
4455
|
-
|
4456
|
-
chan.set_marshaler(injector[ObjMarshalerManager])
|
4457
|
-
|
4458
|
-
#
|
4459
|
-
|
4460
|
-
log_lock = threading.RLock()
|
4461
|
-
send_logs = False
|
4462
|
-
|
4463
|
-
def log_fn(s: str) -> None:
|
4464
|
-
with log_lock:
|
4465
|
-
if send_logs:
|
4466
|
-
chan.send_obj(_RemoteExecutionResponse(l=_RemoteExecutionLog(s)))
|
4467
|
-
|
4468
|
-
log_handler = _RemoteExecutionLogHandler(log_fn)
|
4469
|
-
logging.root.addHandler(log_handler)
|
4470
|
-
|
4471
|
-
#
|
4472
|
-
|
4473
|
-
ce = injector[LocalCommandExecutor]
|
4474
|
-
|
4475
|
-
while True:
|
4476
|
-
req = chan.recv_obj(_RemoteExecutionRequest)
|
4477
|
-
if req is None:
|
4478
|
-
break
|
4479
|
-
|
4480
|
-
with log_lock:
|
4481
|
-
send_logs = True
|
4482
|
-
|
4483
|
-
r = ce.try_execute(
|
4484
|
-
req.c,
|
4485
|
-
log=log,
|
4486
|
-
omit_exc_object=True,
|
4487
|
-
)
|
4488
|
-
|
4489
|
-
with log_lock:
|
4490
|
-
send_logs = False
|
4491
|
-
|
4492
|
-
chan.send_obj(_RemoteExecutionResponse(r=CommandOutputOrExceptionData(
|
4493
|
-
output=r.output,
|
4494
|
-
exception=r.exception,
|
4495
|
-
)))
|
5395
|
+
# ../remote/connection.py
|
4496
5396
|
|
4497
5397
|
|
4498
5398
|
##
|
4499
5399
|
|
4500
5400
|
|
4501
|
-
|
4502
|
-
|
4503
|
-
|
4504
|
-
|
4505
|
-
|
4506
|
-
class RemoteCommandExecutor(CommandExecutor):
|
4507
|
-
def __init__(self, chan: RemoteChannel) -> None:
|
4508
|
-
super().__init__()
|
4509
|
-
|
4510
|
-
self._chan = chan
|
4511
|
-
|
4512
|
-
def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
|
4513
|
-
self._chan.send_obj(_RemoteExecutionRequest(cmd))
|
4514
|
-
|
4515
|
-
while True:
|
4516
|
-
if (r := self._chan.recv_obj(_RemoteExecutionResponse)) is None:
|
4517
|
-
raise EOFError
|
4518
|
-
|
4519
|
-
if r.l is not None:
|
4520
|
-
log.info(r.l.s)
|
4521
|
-
|
4522
|
-
if r.r is not None:
|
4523
|
-
return r.r
|
4524
|
-
|
4525
|
-
# @ta.override
|
4526
|
-
def execute(self, cmd: Command) -> Command.Output:
|
4527
|
-
r = self._remote_execute(cmd)
|
4528
|
-
if (e := r.exception) is not None:
|
4529
|
-
raise RemoteCommandError(e)
|
4530
|
-
else:
|
4531
|
-
return check_not_none(r.output)
|
4532
|
-
|
4533
|
-
# @ta.override
|
4534
|
-
def try_execute(
|
5401
|
+
class RemoteExecutionConnector(abc.ABC):
|
5402
|
+
@abc.abstractmethod
|
5403
|
+
def connect(
|
4535
5404
|
self,
|
4536
|
-
|
4537
|
-
|
4538
|
-
|
4539
|
-
|
4540
|
-
) -> CommandOutputOrException:
|
4541
|
-
try:
|
4542
|
-
r = self._remote_execute(cmd)
|
4543
|
-
|
4544
|
-
except Exception as e: # noqa
|
4545
|
-
if log is not None:
|
4546
|
-
log.exception('Exception executing remote command: %r', type(cmd))
|
4547
|
-
|
4548
|
-
return CommandOutputOrExceptionData(exception=CommandException.of(
|
4549
|
-
e,
|
4550
|
-
omit_exc_object=omit_exc_object,
|
4551
|
-
cmd=cmd,
|
4552
|
-
))
|
4553
|
-
|
4554
|
-
else:
|
4555
|
-
return r
|
5405
|
+
tgt: RemoteSpawning.Target,
|
5406
|
+
bs: MainBootstrap,
|
5407
|
+
) -> ta.AsyncContextManager[RemoteCommandExecutor]:
|
5408
|
+
raise NotImplementedError
|
4556
5409
|
|
4557
5410
|
|
4558
5411
|
##
|
4559
5412
|
|
4560
5413
|
|
4561
|
-
class
|
5414
|
+
class PyremoteRemoteExecutionConnector(RemoteExecutionConnector):
|
4562
5415
|
def __init__(
|
4563
5416
|
self,
|
4564
5417
|
*,
|
@@ -4591,38 +5444,43 @@ class RemoteExecution:
|
|
4591
5444
|
|
4592
5445
|
#
|
4593
5446
|
|
4594
|
-
@contextlib.
|
4595
|
-
def connect(
|
5447
|
+
@contextlib.asynccontextmanager
|
5448
|
+
async def connect(
|
4596
5449
|
self,
|
4597
5450
|
tgt: RemoteSpawning.Target,
|
4598
5451
|
bs: MainBootstrap,
|
4599
|
-
) -> ta.
|
5452
|
+
) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
|
4600
5453
|
spawn_src = self._spawn_src()
|
4601
5454
|
remote_src = self._remote_src()
|
4602
5455
|
|
4603
|
-
with self._spawning.spawn(
|
5456
|
+
async with self._spawning.spawn(
|
4604
5457
|
tgt,
|
4605
5458
|
spawn_src,
|
5459
|
+
debug=bs.main_config.debug,
|
4606
5460
|
) as proc:
|
4607
|
-
res = PyremoteBootstrapDriver( # noqa
|
5461
|
+
res = await PyremoteBootstrapDriver( # noqa
|
4608
5462
|
remote_src,
|
4609
5463
|
PyremoteBootstrapOptions(
|
4610
5464
|
debug=bs.main_config.debug,
|
4611
5465
|
),
|
4612
|
-
).
|
5466
|
+
).async_run(
|
4613
5467
|
proc.stdout,
|
4614
5468
|
proc.stdin,
|
4615
5469
|
)
|
4616
5470
|
|
4617
|
-
chan =
|
5471
|
+
chan = RemoteChannelImpl(
|
4618
5472
|
proc.stdout,
|
4619
5473
|
proc.stdin,
|
4620
5474
|
msh=self._msh,
|
4621
5475
|
)
|
4622
5476
|
|
4623
|
-
chan.send_obj(bs)
|
5477
|
+
await chan.send_obj(bs)
|
5478
|
+
|
5479
|
+
rce: RemoteCommandExecutor
|
5480
|
+
async with contextlib.aclosing(RemoteCommandExecutor(chan)) as rce:
|
5481
|
+
await rce.start()
|
4624
5482
|
|
4625
|
-
|
5483
|
+
yield rce
|
4626
5484
|
|
4627
5485
|
|
4628
5486
|
########################################
|
@@ -4644,7 +5502,6 @@ TODO:
|
|
4644
5502
|
|
4645
5503
|
|
4646
5504
|
class Pyenv:
|
4647
|
-
|
4648
5505
|
def __init__(
|
4649
5506
|
self,
|
4650
5507
|
*,
|
@@ -4657,13 +5514,13 @@ class Pyenv:
|
|
4657
5514
|
|
4658
5515
|
self._root_kw = root
|
4659
5516
|
|
4660
|
-
@
|
4661
|
-
def root(self) -> ta.Optional[str]:
|
5517
|
+
@async_cached_nullary
|
5518
|
+
async def root(self) -> ta.Optional[str]:
|
4662
5519
|
if self._root_kw is not None:
|
4663
5520
|
return self._root_kw
|
4664
5521
|
|
4665
5522
|
if shutil.which('pyenv'):
|
4666
|
-
return
|
5523
|
+
return await asyncio_subprocess_check_output_str('pyenv', 'root')
|
4667
5524
|
|
4668
5525
|
d = os.path.expanduser('~/.pyenv')
|
4669
5526
|
if os.path.isdir(d) and os.path.isfile(os.path.join(d, 'bin', 'pyenv')):
|
@@ -4671,12 +5528,12 @@ class Pyenv:
|
|
4671
5528
|
|
4672
5529
|
return None
|
4673
5530
|
|
4674
|
-
@
|
4675
|
-
def exe(self) -> str:
|
4676
|
-
return os.path.join(check_not_none(self.root()), 'bin', 'pyenv')
|
5531
|
+
@async_cached_nullary
|
5532
|
+
async def exe(self) -> str:
|
5533
|
+
return os.path.join(check_not_none(await self.root()), 'bin', 'pyenv')
|
4677
5534
|
|
4678
|
-
def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
|
4679
|
-
if (root := self.root()) is None:
|
5535
|
+
async def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
|
5536
|
+
if (root := await self.root()) is None:
|
4680
5537
|
return []
|
4681
5538
|
ret = []
|
4682
5539
|
vp = os.path.join(root, 'versions')
|
@@ -4688,11 +5545,11 @@ class Pyenv:
|
|
4688
5545
|
ret.append((dn, ep))
|
4689
5546
|
return ret
|
4690
5547
|
|
4691
|
-
def installable_versions(self) -> ta.List[str]:
|
4692
|
-
if self.root() is None:
|
5548
|
+
async def installable_versions(self) -> ta.List[str]:
|
5549
|
+
if await self.root() is None:
|
4693
5550
|
return []
|
4694
5551
|
ret = []
|
4695
|
-
s =
|
5552
|
+
s = await asyncio_subprocess_check_output_str(await self.exe(), 'install', '--list')
|
4696
5553
|
for l in s.splitlines():
|
4697
5554
|
if not l.startswith(' '):
|
4698
5555
|
continue
|
@@ -4702,12 +5559,12 @@ class Pyenv:
|
|
4702
5559
|
ret.append(l)
|
4703
5560
|
return ret
|
4704
5561
|
|
4705
|
-
def update(self) -> bool:
|
4706
|
-
if (root := self.root()) is None:
|
5562
|
+
async def update(self) -> bool:
|
5563
|
+
if (root := await self.root()) is None:
|
4707
5564
|
return False
|
4708
5565
|
if not os.path.isdir(os.path.join(root, '.git')):
|
4709
5566
|
return False
|
4710
|
-
|
5567
|
+
await asyncio_subprocess_check_call('git', 'pull', cwd=root)
|
4711
5568
|
return True
|
4712
5569
|
|
4713
5570
|
|
@@ -4768,17 +5625,16 @@ THREADED_PYENV_INSTALL_OPTS = PyenvInstallOpts(conf_opts=['--disable-gil'])
|
|
4768
5625
|
|
4769
5626
|
class PyenvInstallOptsProvider(abc.ABC):
|
4770
5627
|
@abc.abstractmethod
|
4771
|
-
def opts(self) -> PyenvInstallOpts:
|
5628
|
+
def opts(self) -> ta.Awaitable[PyenvInstallOpts]:
|
4772
5629
|
raise NotImplementedError
|
4773
5630
|
|
4774
5631
|
|
4775
5632
|
class LinuxPyenvInstallOpts(PyenvInstallOptsProvider):
|
4776
|
-
def opts(self) -> PyenvInstallOpts:
|
5633
|
+
async def opts(self) -> PyenvInstallOpts:
|
4777
5634
|
return PyenvInstallOpts()
|
4778
5635
|
|
4779
5636
|
|
4780
5637
|
class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
4781
|
-
|
4782
5638
|
@cached_nullary
|
4783
5639
|
def framework_opts(self) -> PyenvInstallOpts:
|
4784
5640
|
return PyenvInstallOpts(conf_opts=['--enable-framework'])
|
@@ -4794,12 +5650,12 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
4794
5650
|
'zlib',
|
4795
5651
|
]
|
4796
5652
|
|
4797
|
-
@
|
4798
|
-
def brew_deps_opts(self) -> PyenvInstallOpts:
|
5653
|
+
@async_cached_nullary
|
5654
|
+
async def brew_deps_opts(self) -> PyenvInstallOpts:
|
4799
5655
|
cflags = []
|
4800
5656
|
ldflags = []
|
4801
5657
|
for dep in self.BREW_DEPS:
|
4802
|
-
dep_prefix =
|
5658
|
+
dep_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', dep)
|
4803
5659
|
cflags.append(f'-I{dep_prefix}/include')
|
4804
5660
|
ldflags.append(f'-L{dep_prefix}/lib')
|
4805
5661
|
return PyenvInstallOpts(
|
@@ -4807,13 +5663,13 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
4807
5663
|
ldflags=ldflags,
|
4808
5664
|
)
|
4809
5665
|
|
4810
|
-
@
|
4811
|
-
def brew_tcl_opts(self) -> PyenvInstallOpts:
|
4812
|
-
if
|
5666
|
+
@async_cached_nullary
|
5667
|
+
async def brew_tcl_opts(self) -> PyenvInstallOpts:
|
5668
|
+
if await asyncio_subprocess_try_output('brew', '--prefix', 'tcl-tk') is None:
|
4813
5669
|
return PyenvInstallOpts()
|
4814
5670
|
|
4815
|
-
tcl_tk_prefix =
|
4816
|
-
tcl_tk_ver_str =
|
5671
|
+
tcl_tk_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', 'tcl-tk')
|
5672
|
+
tcl_tk_ver_str = await asyncio_subprocess_check_output_str('brew', 'ls', '--versions', 'tcl-tk')
|
4817
5673
|
tcl_tk_ver = '.'.join(tcl_tk_ver_str.split()[1].split('.')[:2])
|
4818
5674
|
|
4819
5675
|
return PyenvInstallOpts(conf_opts=[
|
@@ -4828,11 +5684,11 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
4828
5684
|
# pkg_config_path += ':' + os.environ['PKG_CONFIG_PATH']
|
4829
5685
|
# return PyenvInstallOpts(env={'PKG_CONFIG_PATH': pkg_config_path})
|
4830
5686
|
|
4831
|
-
def opts(self) -> PyenvInstallOpts:
|
5687
|
+
async def opts(self) -> PyenvInstallOpts:
|
4832
5688
|
return PyenvInstallOpts().merge(
|
4833
5689
|
self.framework_opts(),
|
4834
|
-
self.brew_deps_opts(),
|
4835
|
-
self.brew_tcl_opts(),
|
5690
|
+
await self.brew_deps_opts(),
|
5691
|
+
await self.brew_tcl_opts(),
|
4836
5692
|
# self.brew_ssl_opts(),
|
4837
5693
|
)
|
4838
5694
|
|
@@ -4864,20 +5720,8 @@ class PyenvVersionInstaller:
|
|
4864
5720
|
) -> None:
|
4865
5721
|
super().__init__()
|
4866
5722
|
|
4867
|
-
if no_default_opts:
|
4868
|
-
if opts is None:
|
4869
|
-
opts = PyenvInstallOpts()
|
4870
|
-
else:
|
4871
|
-
lst = [opts if opts is not None else DEFAULT_PYENV_INSTALL_OPTS]
|
4872
|
-
if interp_opts.debug:
|
4873
|
-
lst.append(DEBUG_PYENV_INSTALL_OPTS)
|
4874
|
-
if interp_opts.threaded:
|
4875
|
-
lst.append(THREADED_PYENV_INSTALL_OPTS)
|
4876
|
-
lst.append(PLATFORM_PYENV_INSTALL_OPTS[sys.platform].opts())
|
4877
|
-
opts = PyenvInstallOpts().merge(*lst)
|
4878
|
-
|
4879
5723
|
self._version = version
|
4880
|
-
self.
|
5724
|
+
self._given_opts = opts
|
4881
5725
|
self._interp_opts = interp_opts
|
4882
5726
|
self._given_install_name = install_name
|
4883
5727
|
|
@@ -4888,9 +5732,21 @@ class PyenvVersionInstaller:
|
|
4888
5732
|
def version(self) -> str:
|
4889
5733
|
return self._version
|
4890
5734
|
|
4891
|
-
@
|
4892
|
-
def opts(self) -> PyenvInstallOpts:
|
4893
|
-
|
5735
|
+
@async_cached_nullary
|
5736
|
+
async def opts(self) -> PyenvInstallOpts:
|
5737
|
+
opts = self._given_opts
|
5738
|
+
if self._no_default_opts:
|
5739
|
+
if opts is None:
|
5740
|
+
opts = PyenvInstallOpts()
|
5741
|
+
else:
|
5742
|
+
lst = [self._given_opts if self._given_opts is not None else DEFAULT_PYENV_INSTALL_OPTS]
|
5743
|
+
if self._interp_opts.debug:
|
5744
|
+
lst.append(DEBUG_PYENV_INSTALL_OPTS)
|
5745
|
+
if self._interp_opts.threaded:
|
5746
|
+
lst.append(THREADED_PYENV_INSTALL_OPTS)
|
5747
|
+
lst.append(await PLATFORM_PYENV_INSTALL_OPTS[sys.platform].opts())
|
5748
|
+
opts = PyenvInstallOpts().merge(*lst)
|
5749
|
+
return opts
|
4894
5750
|
|
4895
5751
|
@cached_nullary
|
4896
5752
|
def install_name(self) -> str:
|
@@ -4898,17 +5754,18 @@ class PyenvVersionInstaller:
|
|
4898
5754
|
return self._given_install_name
|
4899
5755
|
return self._version + ('-debug' if self._interp_opts.debug else '')
|
4900
5756
|
|
4901
|
-
@
|
4902
|
-
def install_dir(self) -> str:
|
4903
|
-
return str(os.path.join(check_not_none(self._pyenv.root()), 'versions', self.install_name()))
|
5757
|
+
@async_cached_nullary
|
5758
|
+
async def install_dir(self) -> str:
|
5759
|
+
return str(os.path.join(check_not_none(await self._pyenv.root()), 'versions', self.install_name()))
|
4904
5760
|
|
4905
|
-
@
|
4906
|
-
def install(self) -> str:
|
4907
|
-
|
5761
|
+
@async_cached_nullary
|
5762
|
+
async def install(self) -> str:
|
5763
|
+
opts = await self.opts()
|
5764
|
+
env = {**os.environ, **opts.env}
|
4908
5765
|
for k, l in [
|
4909
|
-
('CFLAGS',
|
4910
|
-
('LDFLAGS',
|
4911
|
-
('PYTHON_CONFIGURE_OPTS',
|
5766
|
+
('CFLAGS', opts.cflags),
|
5767
|
+
('LDFLAGS', opts.ldflags),
|
5768
|
+
('PYTHON_CONFIGURE_OPTS', opts.conf_opts),
|
4912
5769
|
]:
|
4913
5770
|
v = ' '.join(l)
|
4914
5771
|
if k in os.environ:
|
@@ -4916,13 +5773,13 @@ class PyenvVersionInstaller:
|
|
4916
5773
|
env[k] = v
|
4917
5774
|
|
4918
5775
|
conf_args = [
|
4919
|
-
*
|
5776
|
+
*opts.opts,
|
4920
5777
|
self._version,
|
4921
5778
|
]
|
4922
5779
|
|
4923
5780
|
if self._given_install_name is not None:
|
4924
5781
|
full_args = [
|
4925
|
-
os.path.join(check_not_none(self._pyenv.root()), 'plugins', 'python-build', 'bin', 'python-build'),
|
5782
|
+
os.path.join(check_not_none(await self._pyenv.root()), 'plugins', 'python-build', 'bin', 'python-build'), # noqa
|
4926
5783
|
*conf_args,
|
4927
5784
|
self.install_dir(),
|
4928
5785
|
]
|
@@ -4933,12 +5790,12 @@ class PyenvVersionInstaller:
|
|
4933
5790
|
*conf_args,
|
4934
5791
|
]
|
4935
5792
|
|
4936
|
-
|
5793
|
+
await asyncio_subprocess_check_call(
|
4937
5794
|
*full_args,
|
4938
5795
|
env=env,
|
4939
5796
|
)
|
4940
5797
|
|
4941
|
-
exe = os.path.join(self.install_dir(), 'bin', 'python')
|
5798
|
+
exe = os.path.join(await self.install_dir(), 'bin', 'python')
|
4942
5799
|
if not os.path.isfile(exe):
|
4943
5800
|
raise RuntimeError(f'Interpreter not found: {exe}')
|
4944
5801
|
return exe
|
@@ -4948,7 +5805,6 @@ class PyenvVersionInstaller:
|
|
4948
5805
|
|
4949
5806
|
|
4950
5807
|
class PyenvInterpProvider(InterpProvider):
|
4951
|
-
|
4952
5808
|
def __init__(
|
4953
5809
|
self,
|
4954
5810
|
pyenv: Pyenv = Pyenv(),
|
@@ -4991,11 +5847,11 @@ class PyenvInterpProvider(InterpProvider):
|
|
4991
5847
|
exe: str
|
4992
5848
|
version: InterpVersion
|
4993
5849
|
|
4994
|
-
def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
|
5850
|
+
async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
|
4995
5851
|
iv: ta.Optional[InterpVersion]
|
4996
5852
|
if self._inspect:
|
4997
5853
|
try:
|
4998
|
-
iv = check_not_none(self._inspector.inspect(ep)).iv
|
5854
|
+
iv = check_not_none(await self._inspector.inspect(ep)).iv
|
4999
5855
|
except Exception as e: # noqa
|
5000
5856
|
return None
|
5001
5857
|
else:
|
@@ -5008,10 +5864,10 @@ class PyenvInterpProvider(InterpProvider):
|
|
5008
5864
|
version=iv,
|
5009
5865
|
)
|
5010
5866
|
|
5011
|
-
def installed(self) -> ta.Sequence[Installed]:
|
5867
|
+
async def installed(self) -> ta.Sequence[Installed]:
|
5012
5868
|
ret: ta.List[PyenvInterpProvider.Installed] = []
|
5013
|
-
for vn, ep in self._pyenv.version_exes():
|
5014
|
-
if (i := self._make_installed(vn, ep)) is None:
|
5869
|
+
for vn, ep in await self._pyenv.version_exes():
|
5870
|
+
if (i := await self._make_installed(vn, ep)) is None:
|
5015
5871
|
log.debug('Invalid pyenv version: %s', vn)
|
5016
5872
|
continue
|
5017
5873
|
ret.append(i)
|
@@ -5019,11 +5875,11 @@ class PyenvInterpProvider(InterpProvider):
|
|
5019
5875
|
|
5020
5876
|
#
|
5021
5877
|
|
5022
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5023
|
-
return [i.version for i in self.installed()]
|
5878
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5879
|
+
return [i.version for i in await self.installed()]
|
5024
5880
|
|
5025
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5026
|
-
for i in self.installed():
|
5881
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
5882
|
+
for i in await self.installed():
|
5027
5883
|
if i.version == version:
|
5028
5884
|
return Interp(
|
5029
5885
|
exe=i.exe,
|
@@ -5033,10 +5889,10 @@ class PyenvInterpProvider(InterpProvider):
|
|
5033
5889
|
|
5034
5890
|
#
|
5035
5891
|
|
5036
|
-
def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5892
|
+
async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5037
5893
|
lst = []
|
5038
5894
|
|
5039
|
-
for vs in self._pyenv.installable_versions():
|
5895
|
+
for vs in await self._pyenv.installable_versions():
|
5040
5896
|
if (iv := self.guess_version(vs)) is None:
|
5041
5897
|
continue
|
5042
5898
|
if iv.opts.debug:
|
@@ -5046,16 +5902,16 @@ class PyenvInterpProvider(InterpProvider):
|
|
5046
5902
|
|
5047
5903
|
return lst
|
5048
5904
|
|
5049
|
-
def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5050
|
-
lst = self._get_installable_versions(spec)
|
5905
|
+
async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5906
|
+
lst = await self._get_installable_versions(spec)
|
5051
5907
|
|
5052
5908
|
if self._try_update and not any(v in spec for v in lst):
|
5053
5909
|
if self._pyenv.update():
|
5054
|
-
lst = self._get_installable_versions(spec)
|
5910
|
+
lst = await self._get_installable_versions(spec)
|
5055
5911
|
|
5056
5912
|
return lst
|
5057
5913
|
|
5058
|
-
def install_version(self, version: InterpVersion) -> Interp:
|
5914
|
+
async def install_version(self, version: InterpVersion) -> Interp:
|
5059
5915
|
inst_version = str(version.version)
|
5060
5916
|
inst_opts = version.opts
|
5061
5917
|
if inst_opts.threaded:
|
@@ -5067,7 +5923,7 @@ class PyenvInterpProvider(InterpProvider):
|
|
5067
5923
|
interp_opts=inst_opts,
|
5068
5924
|
)
|
5069
5925
|
|
5070
|
-
exe = installer.install()
|
5926
|
+
exe = await installer.install()
|
5071
5927
|
return Interp(exe, version)
|
5072
5928
|
|
5073
5929
|
|
@@ -5146,7 +6002,7 @@ class SystemInterpProvider(InterpProvider):
|
|
5146
6002
|
|
5147
6003
|
#
|
5148
6004
|
|
5149
|
-
def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
|
6005
|
+
async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
|
5150
6006
|
if not self.inspect:
|
5151
6007
|
s = os.path.basename(exe)
|
5152
6008
|
if s.startswith('python'):
|
@@ -5156,13 +6012,13 @@ class SystemInterpProvider(InterpProvider):
|
|
5156
6012
|
return InterpVersion.parse(s)
|
5157
6013
|
except InvalidVersion:
|
5158
6014
|
pass
|
5159
|
-
ii = self.inspector.inspect(exe)
|
6015
|
+
ii = await self.inspector.inspect(exe)
|
5160
6016
|
return ii.iv if ii is not None else None
|
5161
6017
|
|
5162
|
-
def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
|
6018
|
+
async def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
|
5163
6019
|
lst = []
|
5164
6020
|
for e in self.exes():
|
5165
|
-
if (ev := self.get_exe_version(e)) is None:
|
6021
|
+
if (ev := await self.get_exe_version(e)) is None:
|
5166
6022
|
log.debug('Invalid system version: %s', e)
|
5167
6023
|
continue
|
5168
6024
|
lst.append((e, ev))
|
@@ -5170,11 +6026,11 @@ class SystemInterpProvider(InterpProvider):
|
|
5170
6026
|
|
5171
6027
|
#
|
5172
6028
|
|
5173
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5174
|
-
return [ev for e, ev in self.exe_versions()]
|
6029
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
6030
|
+
return [ev for e, ev in await self.exe_versions()]
|
5175
6031
|
|
5176
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5177
|
-
for e, ev in self.exe_versions():
|
6032
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
6033
|
+
for e, ev in await self.exe_versions():
|
5178
6034
|
if ev != version:
|
5179
6035
|
continue
|
5180
6036
|
return Interp(
|
@@ -5195,9 +6051,11 @@ def bind_remote(
|
|
5195
6051
|
lst: ta.List[InjectorBindingOrBindings] = [
|
5196
6052
|
inj.bind(remote_config),
|
5197
6053
|
|
5198
|
-
inj.bind(
|
6054
|
+
inj.bind(SubprocessRemoteSpawning, singleton=True),
|
6055
|
+
inj.bind(RemoteSpawning, to_key=SubprocessRemoteSpawning),
|
5199
6056
|
|
5200
|
-
inj.bind(
|
6057
|
+
inj.bind(PyremoteRemoteExecutionConnector, singleton=True),
|
6058
|
+
inj.bind(RemoteExecutionConnector, to_key=PyremoteRemoteExecutionConnector),
|
5201
6059
|
]
|
5202
6060
|
|
5203
6061
|
if (pf := remote_config.payload_file) is not None:
|
@@ -5221,13 +6079,14 @@ class InterpResolver:
|
|
5221
6079
|
providers: ta.Sequence[ta.Tuple[str, InterpProvider]],
|
5222
6080
|
) -> None:
|
5223
6081
|
super().__init__()
|
6082
|
+
|
5224
6083
|
self._providers: ta.Mapping[str, InterpProvider] = collections.OrderedDict(providers)
|
5225
6084
|
|
5226
|
-
def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
|
6085
|
+
async def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
|
5227
6086
|
lst = [
|
5228
6087
|
(i, si)
|
5229
6088
|
for i, p in enumerate(self._providers.values())
|
5230
|
-
for si in p.get_installed_versions(spec)
|
6089
|
+
for si in await p.get_installed_versions(spec)
|
5231
6090
|
if spec.contains(si)
|
5232
6091
|
]
|
5233
6092
|
|
@@ -5239,16 +6098,16 @@ class InterpResolver:
|
|
5239
6098
|
bp = list(self._providers.values())[bi]
|
5240
6099
|
return (bp, bv)
|
5241
6100
|
|
5242
|
-
def resolve(
|
6101
|
+
async def resolve(
|
5243
6102
|
self,
|
5244
6103
|
spec: InterpSpecifier,
|
5245
6104
|
*,
|
5246
6105
|
install: bool = False,
|
5247
6106
|
) -> ta.Optional[Interp]:
|
5248
|
-
tup = self._resolve_installed(spec)
|
6107
|
+
tup = await self._resolve_installed(spec)
|
5249
6108
|
if tup is not None:
|
5250
6109
|
bp, bv = tup
|
5251
|
-
return bp.get_installed_version(bv)
|
6110
|
+
return await bp.get_installed_version(bv)
|
5252
6111
|
|
5253
6112
|
if not install:
|
5254
6113
|
return None
|
@@ -5256,21 +6115,21 @@ class InterpResolver:
|
|
5256
6115
|
tp = list(self._providers.values())[0] # noqa
|
5257
6116
|
|
5258
6117
|
sv = sorted(
|
5259
|
-
[s for s in tp.get_installable_versions(spec) if s in spec],
|
6118
|
+
[s for s in await tp.get_installable_versions(spec) if s in spec],
|
5260
6119
|
key=lambda s: s.version,
|
5261
6120
|
)
|
5262
6121
|
if not sv:
|
5263
6122
|
return None
|
5264
6123
|
|
5265
6124
|
bv = sv[-1]
|
5266
|
-
return tp.install_version(bv)
|
6125
|
+
return await tp.install_version(bv)
|
5267
6126
|
|
5268
|
-
def list(self, spec: InterpSpecifier) -> None:
|
6127
|
+
async def list(self, spec: InterpSpecifier) -> None:
|
5269
6128
|
print('installed:')
|
5270
6129
|
for n, p in self._providers.items():
|
5271
6130
|
lst = [
|
5272
6131
|
si
|
5273
|
-
for si in p.get_installed_versions(spec)
|
6132
|
+
for si in await p.get_installed_versions(spec)
|
5274
6133
|
if spec.contains(si)
|
5275
6134
|
]
|
5276
6135
|
if lst:
|
@@ -5284,7 +6143,7 @@ class InterpResolver:
|
|
5284
6143
|
for n, p in self._providers.items():
|
5285
6144
|
lst = [
|
5286
6145
|
si
|
5287
|
-
for si in p.get_installable_versions(spec)
|
6146
|
+
for si in await p.get_installable_versions(spec)
|
5288
6147
|
if spec.contains(si)
|
5289
6148
|
]
|
5290
6149
|
if lst:
|
@@ -5326,9 +6185,9 @@ class InterpCommand(Command['InterpCommand.Output']):
|
|
5326
6185
|
|
5327
6186
|
|
5328
6187
|
class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
|
5329
|
-
def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
|
6188
|
+
async def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
|
5330
6189
|
i = InterpSpecifier.parse(check_not_none(cmd.spec))
|
5331
|
-
o = check_not_none(DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
|
6190
|
+
o = check_not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
|
5332
6191
|
return InterpCommand.Output(
|
5333
6192
|
exe=o.exe,
|
5334
6193
|
version=str(o.version.version),
|
@@ -5367,7 +6226,7 @@ def bind_command(
|
|
5367
6226
|
class _FactoryCommandExecutor(CommandExecutor):
|
5368
6227
|
factory: ta.Callable[[], CommandExecutor]
|
5369
6228
|
|
5370
|
-
def execute(self, i: Command) -> Command.Output:
|
6229
|
+
def execute(self, i: Command) -> ta.Awaitable[Command.Output]:
|
5371
6230
|
return self.factory().execute(i)
|
5372
6231
|
|
5373
6232
|
|
@@ -5522,31 +6381,7 @@ def main_bootstrap(bs: MainBootstrap) -> Injector:
|
|
5522
6381
|
##
|
5523
6382
|
|
5524
6383
|
|
5525
|
-
def
|
5526
|
-
import argparse
|
5527
|
-
|
5528
|
-
parser = argparse.ArgumentParser()
|
5529
|
-
|
5530
|
-
parser.add_argument('--_payload-file')
|
5531
|
-
|
5532
|
-
parser.add_argument('-s', '--shell')
|
5533
|
-
parser.add_argument('-q', '--shell-quote', action='store_true')
|
5534
|
-
parser.add_argument('--python', default='python3')
|
5535
|
-
|
5536
|
-
parser.add_argument('--pycharm-debug-port', type=int)
|
5537
|
-
parser.add_argument('--pycharm-debug-host')
|
5538
|
-
parser.add_argument('--pycharm-debug-version')
|
5539
|
-
|
5540
|
-
parser.add_argument('--debug', action='store_true')
|
5541
|
-
|
5542
|
-
parser.add_argument('--local', action='store_true')
|
5543
|
-
|
5544
|
-
parser.add_argument('command', nargs='+')
|
5545
|
-
|
5546
|
-
args = parser.parse_args()
|
5547
|
-
|
5548
|
-
#
|
5549
|
-
|
6384
|
+
async def _async_main(args: ta.Any) -> None:
|
5550
6385
|
bs = MainBootstrap(
|
5551
6386
|
main_config=MainConfig(
|
5552
6387
|
log_level='DEBUG' if args.debug else 'INFO',
|
@@ -5559,12 +6394,16 @@ def _main() -> None:
|
|
5559
6394
|
|
5560
6395
|
pycharm_remote_debug=PycharmRemoteDebug(
|
5561
6396
|
port=args.pycharm_debug_port,
|
5562
|
-
host=args.pycharm_debug_host,
|
6397
|
+
**(dict(host=args.pycharm_debug_host) if args.pycharm_debug_host is not None else {}),
|
5563
6398
|
install_version=args.pycharm_debug_version,
|
5564
6399
|
) if args.pycharm_debug_port is not None else None,
|
6400
|
+
|
6401
|
+
timebomb_delay_s=args.remote_timebomb_delay_s,
|
5565
6402
|
),
|
5566
6403
|
)
|
5567
6404
|
|
6405
|
+
#
|
6406
|
+
|
5568
6407
|
injector = main_bootstrap(
|
5569
6408
|
bs,
|
5570
6409
|
)
|
@@ -5583,7 +6422,7 @@ def _main() -> None:
|
|
5583
6422
|
|
5584
6423
|
#
|
5585
6424
|
|
5586
|
-
with contextlib.
|
6425
|
+
async with contextlib.AsyncExitStack() as es:
|
5587
6426
|
ce: CommandExecutor
|
5588
6427
|
|
5589
6428
|
if args.local:
|
@@ -5596,16 +6435,51 @@ def _main() -> None:
|
|
5596
6435
|
python=args.python,
|
5597
6436
|
)
|
5598
6437
|
|
5599
|
-
ce = es.
|
6438
|
+
ce = await es.enter_async_context(injector[RemoteExecutionConnector].connect(tgt, bs)) # noqa
|
5600
6439
|
|
5601
|
-
|
5602
|
-
|
6440
|
+
async def run_command(cmd: Command) -> None:
|
6441
|
+
res = await ce.try_execute(
|
5603
6442
|
cmd,
|
5604
6443
|
log=log,
|
5605
6444
|
omit_exc_object=True,
|
5606
6445
|
)
|
5607
6446
|
|
5608
|
-
print(msh.marshal_obj(
|
6447
|
+
print(msh.marshal_obj(res, opts=ObjMarshalOptions(raw_bytes=True)))
|
6448
|
+
|
6449
|
+
await asyncio.gather(*[
|
6450
|
+
run_command(cmd)
|
6451
|
+
for cmd in cmds
|
6452
|
+
])
|
6453
|
+
|
6454
|
+
|
6455
|
+
def _main() -> None:
|
6456
|
+
import argparse
|
6457
|
+
|
6458
|
+
parser = argparse.ArgumentParser()
|
6459
|
+
|
6460
|
+
parser.add_argument('--_payload-file')
|
6461
|
+
|
6462
|
+
parser.add_argument('-s', '--shell')
|
6463
|
+
parser.add_argument('-q', '--shell-quote', action='store_true')
|
6464
|
+
parser.add_argument('--python', default='python3')
|
6465
|
+
|
6466
|
+
parser.add_argument('--pycharm-debug-port', type=int)
|
6467
|
+
parser.add_argument('--pycharm-debug-host')
|
6468
|
+
parser.add_argument('--pycharm-debug-version')
|
6469
|
+
|
6470
|
+
parser.add_argument('--remote-timebomb-delay-s', type=float)
|
6471
|
+
|
6472
|
+
parser.add_argument('--debug', action='store_true')
|
6473
|
+
|
6474
|
+
parser.add_argument('--local', action='store_true')
|
6475
|
+
|
6476
|
+
parser.add_argument('command', nargs='+')
|
6477
|
+
|
6478
|
+
args = parser.parse_args()
|
6479
|
+
|
6480
|
+
#
|
6481
|
+
|
6482
|
+
asyncio.run(_async_main(args))
|
5609
6483
|
|
5610
6484
|
|
5611
6485
|
if __name__ == '__main__':
|