ominfra 0.0.0.dev147__py3-none-any.whl → 0.0.0.dev149__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/cli.py +1 -1
- ominfra/configs.py +30 -5
- ominfra/journald/messages.py +1 -1
- ominfra/manage/commands/base.py +5 -5
- ominfra/manage/commands/execution.py +2 -2
- ominfra/manage/commands/inject.py +1 -1
- ominfra/manage/commands/interp.py +2 -2
- ominfra/manage/commands/subprocess.py +22 -14
- ominfra/manage/deploy/command.py +1 -1
- ominfra/manage/deploy/paths.py +2 -2
- ominfra/manage/main.py +48 -32
- ominfra/manage/remote/_main.py +172 -0
- ominfra/manage/remote/channel.py +41 -16
- ominfra/manage/remote/config.py +10 -0
- ominfra/manage/remote/connection.py +106 -0
- ominfra/manage/remote/execution.py +244 -155
- ominfra/manage/remote/inject.py +7 -3
- ominfra/manage/remote/spawning.py +51 -33
- ominfra/pyremote.py +28 -3
- ominfra/scripts/journald2aws.py +195 -91
- ominfra/scripts/manage.py +1366 -486
- ominfra/scripts/supervisor.py +533 -479
- ominfra/supervisor/dispatchers.py +1 -1
- ominfra/supervisor/http.py +2 -2
- ominfra/supervisor/inject.py +4 -4
- ominfra/supervisor/io.py +1 -1
- ominfra/supervisor/spawningimpl.py +1 -1
- ominfra/supervisor/supervisor.py +1 -1
- ominfra/supervisor/types.py +1 -1
- ominfra/tailscale/cli.py +1 -1
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/RECORD +36 -34
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -3,16 +3,20 @@
|
|
3
3
|
# @omlish-lite
|
4
4
|
# @omlish-script
|
5
5
|
# @omlish-amalg-output ../manage/main.py
|
6
|
-
# ruff: noqa: N802 UP006 UP007 UP036
|
6
|
+
# ruff: noqa: N802 TC003 UP006 UP007 UP036
|
7
7
|
"""
|
8
8
|
manage.py -s 'docker run -i python:3.12'
|
9
9
|
manage.py -s 'ssh -i /foo/bar.pem foo@bar.baz' -q --python=python3.8
|
10
10
|
"""
|
11
11
|
import abc
|
12
|
+
import asyncio
|
13
|
+
import asyncio.base_subprocess
|
14
|
+
import asyncio.subprocess
|
12
15
|
import base64
|
13
16
|
import collections
|
14
17
|
import collections.abc
|
15
18
|
import contextlib
|
19
|
+
import ctypes as ct
|
16
20
|
import dataclasses as dc
|
17
21
|
import datetime
|
18
22
|
import decimal
|
@@ -30,6 +34,7 @@ import pwd
|
|
30
34
|
import re
|
31
35
|
import shlex
|
32
36
|
import shutil
|
37
|
+
import signal
|
33
38
|
import site
|
34
39
|
import struct
|
35
40
|
import subprocess
|
@@ -62,6 +67,9 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
|
|
62
67
|
VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType] # noqa
|
63
68
|
VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
|
64
69
|
|
70
|
+
# ../../omlish/lite/asyncio/asyncio.py
|
71
|
+
AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
|
72
|
+
|
65
73
|
# ../../omlish/lite/cached.py
|
66
74
|
T = ta.TypeVar('T')
|
67
75
|
CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
|
@@ -822,10 +830,8 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
822
830
|
with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
|
823
831
|
main_src = sf.read()
|
824
832
|
|
825
|
-
# Restore
|
833
|
+
# Restore vars
|
826
834
|
sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
|
827
|
-
|
828
|
-
# Grab context name
|
829
835
|
context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
|
830
836
|
|
831
837
|
# Write third ack
|
@@ -1000,12 +1006,102 @@ class PyremoteBootstrapDriver:
|
|
1000
1006
|
else:
|
1001
1007
|
raise TypeError(go)
|
1002
1008
|
|
1009
|
+
async def async_run(
|
1010
|
+
self,
|
1011
|
+
input: ta.Any, # asyncio.StreamWriter # noqa
|
1012
|
+
output: ta.Any, # asyncio.StreamReader
|
1013
|
+
) -> Result:
|
1014
|
+
gen = self.gen()
|
1015
|
+
|
1016
|
+
gi: ta.Optional[bytes] = None
|
1017
|
+
while True:
|
1018
|
+
try:
|
1019
|
+
if gi is not None:
|
1020
|
+
go = gen.send(gi)
|
1021
|
+
else:
|
1022
|
+
go = next(gen)
|
1023
|
+
except StopIteration as e:
|
1024
|
+
return e.value
|
1025
|
+
|
1026
|
+
if isinstance(go, self.Read):
|
1027
|
+
if len(gi := await input.read(go.sz)) != go.sz:
|
1028
|
+
raise EOFError
|
1029
|
+
elif isinstance(go, self.Write):
|
1030
|
+
gi = None
|
1031
|
+
output.write(go.d)
|
1032
|
+
await output.drain()
|
1033
|
+
else:
|
1034
|
+
raise TypeError(go)
|
1035
|
+
|
1036
|
+
|
1037
|
+
########################################
|
1038
|
+
# ../../../omlish/lite/asyncio/asyncio.py
|
1039
|
+
|
1040
|
+
|
1041
|
+
##
|
1042
|
+
|
1043
|
+
|
1044
|
+
ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
|
1045
|
+
|
1046
|
+
|
1047
|
+
async def asyncio_open_stream_reader(
|
1048
|
+
f: ta.IO,
|
1049
|
+
loop: ta.Any = None,
|
1050
|
+
*,
|
1051
|
+
limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
|
1052
|
+
) -> asyncio.StreamReader:
|
1053
|
+
if loop is None:
|
1054
|
+
loop = asyncio.get_running_loop()
|
1055
|
+
|
1056
|
+
reader = asyncio.StreamReader(limit=limit, loop=loop)
|
1057
|
+
await loop.connect_read_pipe(
|
1058
|
+
lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
|
1059
|
+
f,
|
1060
|
+
)
|
1061
|
+
|
1062
|
+
return reader
|
1063
|
+
|
1064
|
+
|
1065
|
+
async def asyncio_open_stream_writer(
|
1066
|
+
f: ta.IO,
|
1067
|
+
loop: ta.Any = None,
|
1068
|
+
) -> asyncio.StreamWriter:
|
1069
|
+
if loop is None:
|
1070
|
+
loop = asyncio.get_running_loop()
|
1071
|
+
|
1072
|
+
writer_transport, writer_protocol = await loop.connect_write_pipe(
|
1073
|
+
lambda: asyncio.streams.FlowControlMixin(loop=loop),
|
1074
|
+
f,
|
1075
|
+
)
|
1076
|
+
|
1077
|
+
return asyncio.streams.StreamWriter(
|
1078
|
+
writer_transport,
|
1079
|
+
writer_protocol,
|
1080
|
+
None,
|
1081
|
+
loop,
|
1082
|
+
)
|
1083
|
+
|
1084
|
+
|
1085
|
+
##
|
1086
|
+
|
1087
|
+
|
1088
|
+
def asyncio_maybe_timeout(
|
1089
|
+
fut: AwaitableT,
|
1090
|
+
timeout: ta.Optional[float] = None,
|
1091
|
+
) -> AwaitableT:
|
1092
|
+
if timeout is not None:
|
1093
|
+
fut = asyncio.wait_for(fut, timeout) # type: ignore
|
1094
|
+
return fut
|
1095
|
+
|
1003
1096
|
|
1004
1097
|
########################################
|
1005
1098
|
# ../../../omlish/lite/cached.py
|
1006
1099
|
|
1007
1100
|
|
1008
|
-
|
1101
|
+
##
|
1102
|
+
|
1103
|
+
|
1104
|
+
class _AbstractCachedNullary:
|
1009
1105
|
def __init__(self, fn):
|
1010
1106
|
super().__init__()
|
1011
1107
|
self._fn = fn
|
@@ -1013,17 +1109,25 @@ class _cached_nullary: # noqa
|
|
1013
1109
|
functools.update_wrapper(self, fn)
|
1014
1110
|
|
1015
1111
|
def __call__(self, *args, **kwargs): # noqa
|
1016
|
-
|
1017
|
-
self._value = self._fn()
|
1018
|
-
return self._value
|
1112
|
+
raise TypeError
|
1019
1113
|
|
1020
1114
|
def __get__(self, instance, owner): # noqa
|
1021
1115
|
bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
|
1022
1116
|
return bound
|
1023
1117
|
|
1024
1118
|
|
1119
|
+
##
|
1120
|
+
|
1121
|
+
|
1122
|
+
class _CachedNullary(_AbstractCachedNullary):
|
1123
|
+
def __call__(self, *args, **kwargs): # noqa
|
1124
|
+
if self._value is self._missing:
|
1125
|
+
self._value = self._fn()
|
1126
|
+
return self._value
|
1127
|
+
|
1128
|
+
|
1025
1129
|
def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
1026
|
-
return
|
1130
|
+
return _CachedNullary(fn)
|
1027
1131
|
|
1028
1132
|
|
1029
1133
|
def static_init(fn: CallableT) -> CallableT:
|
@@ -1032,6 +1136,20 @@ def static_init(fn: CallableT) -> CallableT:
|
|
1032
1136
|
return fn
|
1033
1137
|
|
1034
1138
|
|
1139
|
+
##
|
1140
|
+
|
1141
|
+
|
1142
|
+
class _AsyncCachedNullary(_AbstractCachedNullary):
|
1143
|
+
async def __call__(self, *args, **kwargs):
|
1144
|
+
if self._value is self._missing:
|
1145
|
+
self._value = await self._fn()
|
1146
|
+
return self._value
|
1147
|
+
|
1148
|
+
|
1149
|
+
def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
1150
|
+
return _AsyncCachedNullary(fn)
|
1151
|
+
|
1152
|
+
|
1035
1153
|
########################################
|
1036
1154
|
# ../../../omlish/lite/check.py
|
1037
1155
|
|
@@ -1071,6 +1189,11 @@ def check_non_empty_str(v: ta.Optional[str]) -> str:
|
|
1071
1189
|
return v
|
1072
1190
|
|
1073
1191
|
|
1192
|
+
def check_arg(v: bool, msg: str = 'Illegal argument') -> None:
|
1193
|
+
if not v:
|
1194
|
+
raise ValueError(msg)
|
1195
|
+
|
1196
|
+
|
1074
1197
|
def check_state(v: bool, msg: str = 'Illegal state') -> None:
|
1075
1198
|
if not v:
|
1076
1199
|
raise ValueError(msg)
|
@@ -1123,12 +1246,36 @@ def check_empty(v: SizedT) -> SizedT:
|
|
1123
1246
|
return v
|
1124
1247
|
|
1125
1248
|
|
1126
|
-
def
|
1249
|
+
def check_not_empty(v: SizedT) -> SizedT:
|
1127
1250
|
if not len(v):
|
1128
1251
|
raise ValueError(v)
|
1129
1252
|
return v
|
1130
1253
|
|
1131
1254
|
|
1255
|
+
########################################
|
1256
|
+
# ../../../omlish/lite/deathsig.py
|
1257
|
+
|
1258
|
+
|
1259
|
+
LINUX_PR_SET_PDEATHSIG = 1 # Second arg is a signal
|
1260
|
+
LINUX_PR_GET_PDEATHSIG = 2 # Second arg is a ptr to return the signal
|
1261
|
+
|
1262
|
+
|
1263
|
+
def set_process_deathsig(sig: int) -> bool:
|
1264
|
+
if sys.platform == 'linux':
|
1265
|
+
libc = ct.CDLL('libc.so.6')
|
1266
|
+
|
1267
|
+
# int prctl(int option, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5);
|
1268
|
+
libc.prctl.restype = ct.c_int
|
1269
|
+
libc.prctl.argtypes = [ct.c_int, ct.c_ulong, ct.c_ulong, ct.c_ulong, ct.c_ulong]
|
1270
|
+
|
1271
|
+
libc.prctl(LINUX_PR_SET_PDEATHSIG, sig, 0, 0, 0, 0)
|
1272
|
+
|
1273
|
+
return True
|
1274
|
+
|
1275
|
+
else:
|
1276
|
+
return False
|
1277
|
+
|
1278
|
+
|
1132
1279
|
########################################
|
1133
1280
|
# ../../../omlish/lite/json.py
|
1134
1281
|
|
@@ -1910,8 +2057,8 @@ class Command(abc.ABC, ta.Generic[CommandOutputT]):
|
|
1910
2057
|
pass
|
1911
2058
|
|
1912
2059
|
@ta.final
|
1913
|
-
def execute(self, executor: 'CommandExecutor') -> CommandOutputT:
|
1914
|
-
return check_isinstance(executor.execute(self), self.Output) # type: ignore[return-value]
|
2060
|
+
async def execute(self, executor: 'CommandExecutor') -> CommandOutputT:
|
2061
|
+
return check_isinstance(await executor.execute(self), self.Output) # type: ignore[return-value]
|
1915
2062
|
|
1916
2063
|
|
1917
2064
|
##
|
@@ -1972,10 +2119,10 @@ class CommandOutputOrExceptionData(CommandOutputOrException):
|
|
1972
2119
|
|
1973
2120
|
class CommandExecutor(abc.ABC, ta.Generic[CommandT, CommandOutputT]):
|
1974
2121
|
@abc.abstractmethod
|
1975
|
-
def execute(self, cmd: CommandT) -> CommandOutputT:
|
2122
|
+
def execute(self, cmd: CommandT) -> ta.Awaitable[CommandOutputT]:
|
1976
2123
|
raise NotImplementedError
|
1977
2124
|
|
1978
|
-
def try_execute(
|
2125
|
+
async def try_execute(
|
1979
2126
|
self,
|
1980
2127
|
cmd: CommandT,
|
1981
2128
|
*,
|
@@ -1983,7 +2130,7 @@ class CommandExecutor(abc.ABC, ta.Generic[CommandT, CommandOutputT]):
|
|
1983
2130
|
omit_exc_object: bool = False,
|
1984
2131
|
) -> CommandOutputOrException[CommandOutputT]:
|
1985
2132
|
try:
|
1986
|
-
o = self.execute(cmd)
|
2133
|
+
o = await self.execute(cmd)
|
1987
2134
|
|
1988
2135
|
except Exception as e: # noqa
|
1989
2136
|
if log is not None:
|
@@ -2054,8 +2201,18 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
|
|
2054
2201
|
class RemoteConfig:
|
2055
2202
|
payload_file: ta.Optional[str] = None
|
2056
2203
|
|
2204
|
+
set_pgid: bool = True
|
2205
|
+
|
2206
|
+
deathsig: ta.Optional[str] = 'KILL'
|
2207
|
+
|
2057
2208
|
pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
|
2058
2209
|
|
2210
|
+
forward_logging: bool = True
|
2211
|
+
|
2212
|
+
timebomb_delay_s: ta.Optional[float] = 60 * 60.
|
2213
|
+
|
2214
|
+
heartbeat_interval_s: float = 3.
|
2215
|
+
|
2059
2216
|
|
2060
2217
|
########################################
|
2061
2218
|
# ../remote/payload.py
|
@@ -3788,6 +3945,8 @@ class InterpSpecifier:
|
|
3788
3945
|
s, o = InterpOpts.parse_suffix(s)
|
3789
3946
|
if not any(s.startswith(o) for o in Specifier.OPERATORS):
|
3790
3947
|
s = '~=' + s
|
3948
|
+
if s.count('.') < 2:
|
3949
|
+
s += '.0'
|
3791
3950
|
return cls(
|
3792
3951
|
specifier=Specifier(s),
|
3793
3952
|
opts=o,
|
@@ -3834,9 +3993,9 @@ class LocalCommandExecutor(CommandExecutor):
|
|
3834
3993
|
|
3835
3994
|
self._command_executors = command_executors
|
3836
3995
|
|
3837
|
-
def execute(self, cmd: Command) -> Command.Output:
|
3996
|
+
async def execute(self, cmd: Command) -> Command.Output:
|
3838
3997
|
ce: CommandExecutor = self._command_executors[type(cmd)]
|
3839
|
-
return ce.execute(cmd)
|
3998
|
+
return await ce.execute(cmd)
|
3840
3999
|
|
3841
4000
|
|
3842
4001
|
########################################
|
@@ -3882,7 +4041,7 @@ class DeployCommand(Command['DeployCommand.Output']):
|
|
3882
4041
|
|
3883
4042
|
|
3884
4043
|
class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
|
3885
|
-
def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
4044
|
+
async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
3886
4045
|
log.info('Deploying!')
|
3887
4046
|
|
3888
4047
|
return DeployCommand.Output()
|
@@ -3904,11 +4063,30 @@ ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMar
|
|
3904
4063
|
# ../remote/channel.py
|
3905
4064
|
|
3906
4065
|
|
3907
|
-
|
4066
|
+
##
|
4067
|
+
|
4068
|
+
|
4069
|
+
class RemoteChannel(abc.ABC):
|
4070
|
+
@abc.abstractmethod
|
4071
|
+
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Awaitable[None]:
|
4072
|
+
raise NotImplementedError
|
4073
|
+
|
4074
|
+
@abc.abstractmethod
|
4075
|
+
def recv_obj(self, ty: ta.Type[T]) -> ta.Awaitable[ta.Optional[T]]:
|
4076
|
+
raise NotImplementedError
|
4077
|
+
|
4078
|
+
def set_marshaler(self, msh: ObjMarshalerManager) -> None: # noqa
|
4079
|
+
pass
|
4080
|
+
|
4081
|
+
|
4082
|
+
##
|
4083
|
+
|
4084
|
+
|
4085
|
+
class RemoteChannelImpl(RemoteChannel):
|
3908
4086
|
def __init__(
|
3909
4087
|
self,
|
3910
|
-
input:
|
3911
|
-
output:
|
4088
|
+
input: asyncio.StreamReader, # noqa
|
4089
|
+
output: asyncio.StreamWriter,
|
3912
4090
|
*,
|
3913
4091
|
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
3914
4092
|
) -> None:
|
@@ -3918,41 +4096,46 @@ class RemoteChannel:
|
|
3918
4096
|
self._output = output
|
3919
4097
|
self._msh = msh
|
3920
4098
|
|
3921
|
-
self.
|
4099
|
+
self._input_lock = asyncio.Lock()
|
4100
|
+
self._output_lock = asyncio.Lock()
|
3922
4101
|
|
3923
4102
|
def set_marshaler(self, msh: ObjMarshalerManager) -> None:
|
3924
4103
|
self._msh = msh
|
3925
4104
|
|
3926
|
-
|
4105
|
+
#
|
4106
|
+
|
4107
|
+
async def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
3927
4108
|
j = json_dumps_compact(self._msh.marshal_obj(o, ty))
|
3928
4109
|
d = j.encode('utf-8')
|
3929
4110
|
|
3930
4111
|
self._output.write(struct.pack('<I', len(d)))
|
3931
4112
|
self._output.write(d)
|
3932
|
-
self._output.
|
4113
|
+
await self._output.drain()
|
3933
4114
|
|
3934
|
-
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
3935
|
-
with self.
|
3936
|
-
return self._send_obj(o, ty)
|
4115
|
+
async def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
4116
|
+
async with self._output_lock:
|
4117
|
+
return await self._send_obj(o, ty)
|
4118
|
+
|
4119
|
+
#
|
3937
4120
|
|
3938
|
-
def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
3939
|
-
d = self._input.read(4)
|
4121
|
+
async def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
4122
|
+
d = await self._input.read(4)
|
3940
4123
|
if not d:
|
3941
4124
|
return None
|
3942
4125
|
if len(d) != 4:
|
3943
4126
|
raise EOFError
|
3944
4127
|
|
3945
4128
|
sz = struct.unpack('<I', d)[0]
|
3946
|
-
d = self._input.read(sz)
|
4129
|
+
d = await self._input.read(sz)
|
3947
4130
|
if len(d) != sz:
|
3948
4131
|
raise EOFError
|
3949
4132
|
|
3950
4133
|
j = json.loads(d.decode('utf-8'))
|
3951
4134
|
return self._msh.unmarshal_obj(j, ty)
|
3952
4135
|
|
3953
|
-
def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
3954
|
-
with self.
|
3955
|
-
return self._recv_obj(ty)
|
4136
|
+
async def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
4137
|
+
async with self._input_lock:
|
4138
|
+
return await self._recv_obj(ty)
|
3956
4139
|
|
3957
4140
|
|
3958
4141
|
########################################
|
@@ -3986,7 +4169,7 @@ def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
|
|
3986
4169
|
return args
|
3987
4170
|
|
3988
4171
|
|
3989
|
-
def
|
4172
|
+
def prepare_subprocess_invocation(
|
3990
4173
|
*args: str,
|
3991
4174
|
env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
3992
4175
|
extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
@@ -3994,9 +4177,9 @@ def _prepare_subprocess_invocation(
|
|
3994
4177
|
shell: bool = False,
|
3995
4178
|
**kwargs: ta.Any,
|
3996
4179
|
) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
|
3997
|
-
log.debug(args)
|
4180
|
+
log.debug('prepare_subprocess_invocation: args=%r', args)
|
3998
4181
|
if extra_env:
|
3999
|
-
log.debug(extra_env)
|
4182
|
+
log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
|
4000
4183
|
|
4001
4184
|
if extra_env:
|
4002
4185
|
env = {**(env if env is not None else os.environ), **extra_env}
|
@@ -4015,14 +4198,46 @@ def _prepare_subprocess_invocation(
|
|
4015
4198
|
)
|
4016
4199
|
|
4017
4200
|
|
4018
|
-
|
4019
|
-
|
4020
|
-
|
4201
|
+
##
|
4202
|
+
|
4203
|
+
|
4204
|
+
@contextlib.contextmanager
|
4205
|
+
def subprocess_common_context(*args: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
|
4206
|
+
start_time = time.time()
|
4207
|
+
try:
|
4208
|
+
log.debug('subprocess_common_context.try: args=%r', args)
|
4209
|
+
yield
|
4210
|
+
|
4211
|
+
except Exception as exc: # noqa
|
4212
|
+
log.debug('subprocess_common_context.except: exc=%r', exc)
|
4213
|
+
raise
|
4214
|
+
|
4215
|
+
finally:
|
4216
|
+
end_time = time.time()
|
4217
|
+
elapsed_s = end_time - start_time
|
4218
|
+
log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
|
4219
|
+
|
4220
|
+
|
4221
|
+
##
|
4222
|
+
|
4223
|
+
|
4224
|
+
def subprocess_check_call(
|
4225
|
+
*args: str,
|
4226
|
+
stdout: ta.Any = sys.stderr,
|
4227
|
+
**kwargs: ta.Any,
|
4228
|
+
) -> None:
|
4229
|
+
args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
|
4230
|
+
with subprocess_common_context(*args, **kwargs):
|
4231
|
+
return subprocess.check_call(args, **kwargs) # type: ignore
|
4021
4232
|
|
4022
4233
|
|
4023
|
-
def subprocess_check_output(
|
4024
|
-
|
4025
|
-
|
4234
|
+
def subprocess_check_output(
|
4235
|
+
*args: str,
|
4236
|
+
**kwargs: ta.Any,
|
4237
|
+
) -> bytes:
|
4238
|
+
args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
|
4239
|
+
with subprocess_common_context(*args, **kwargs):
|
4240
|
+
return subprocess.check_output(args, **kwargs)
|
4026
4241
|
|
4027
4242
|
|
4028
4243
|
def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
|
@@ -4038,16 +4253,31 @@ DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
|
|
4038
4253
|
)
|
4039
4254
|
|
4040
4255
|
|
4041
|
-
def
|
4042
|
-
|
4256
|
+
def _subprocess_try_run(
|
4257
|
+
fn: ta.Callable[..., T],
|
4258
|
+
*args: ta.Any,
|
4043
4259
|
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4044
4260
|
**kwargs: ta.Any,
|
4045
|
-
) ->
|
4261
|
+
) -> ta.Union[T, Exception]:
|
4046
4262
|
try:
|
4047
|
-
|
4263
|
+
return fn(*args, **kwargs)
|
4048
4264
|
except try_exceptions as e: # noqa
|
4049
4265
|
if log.isEnabledFor(logging.DEBUG):
|
4050
4266
|
log.exception('command failed')
|
4267
|
+
return e
|
4268
|
+
|
4269
|
+
|
4270
|
+
def subprocess_try_call(
|
4271
|
+
*args: str,
|
4272
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4273
|
+
**kwargs: ta.Any,
|
4274
|
+
) -> bool:
|
4275
|
+
if isinstance(_subprocess_try_run(
|
4276
|
+
subprocess_check_call,
|
4277
|
+
*args,
|
4278
|
+
try_exceptions=try_exceptions,
|
4279
|
+
**kwargs,
|
4280
|
+
), Exception):
|
4051
4281
|
return False
|
4052
4282
|
else:
|
4053
4283
|
return True
|
@@ -4058,12 +4288,15 @@ def subprocess_try_output(
|
|
4058
4288
|
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4059
4289
|
**kwargs: ta.Any,
|
4060
4290
|
) -> ta.Optional[bytes]:
|
4061
|
-
|
4062
|
-
|
4063
|
-
|
4064
|
-
|
4065
|
-
|
4291
|
+
if isinstance(ret := _subprocess_try_run(
|
4292
|
+
subprocess_check_output,
|
4293
|
+
*args,
|
4294
|
+
try_exceptions=try_exceptions,
|
4295
|
+
**kwargs,
|
4296
|
+
), Exception):
|
4066
4297
|
return None
|
4298
|
+
else:
|
4299
|
+
return ret
|
4067
4300
|
|
4068
4301
|
|
4069
4302
|
def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|
@@ -4090,175 +4323,922 @@ def subprocess_close(
|
|
4090
4323
|
|
4091
4324
|
|
4092
4325
|
########################################
|
4093
|
-
#
|
4326
|
+
# ../remote/execution.py
|
4094
4327
|
|
4095
4328
|
|
4096
|
-
|
4097
|
-
class InterpInspection:
|
4098
|
-
exe: str
|
4099
|
-
version: Version
|
4329
|
+
##
|
4100
4330
|
|
4101
|
-
version_str: str
|
4102
|
-
config_vars: ta.Mapping[str, str]
|
4103
|
-
prefix: str
|
4104
|
-
base_prefix: str
|
4105
4331
|
|
4106
|
-
|
4107
|
-
|
4108
|
-
|
4109
|
-
|
4110
|
-
debug=bool(self.config_vars.get('Py_DEBUG')),
|
4111
|
-
)
|
4332
|
+
class _RemoteProtocol:
|
4333
|
+
class Message(abc.ABC): # noqa
|
4334
|
+
async def send(self, chan: RemoteChannel) -> None:
|
4335
|
+
await chan.send_obj(self, _RemoteProtocol.Message)
|
4112
4336
|
|
4113
|
-
|
4114
|
-
|
4115
|
-
|
4116
|
-
version=self.version,
|
4117
|
-
opts=self.opts,
|
4118
|
-
)
|
4337
|
+
@classmethod
|
4338
|
+
async def recv(cls: ta.Type[T], chan: RemoteChannel) -> ta.Optional[T]:
|
4339
|
+
return await chan.recv_obj(cls)
|
4119
4340
|
|
4120
|
-
|
4121
|
-
def is_venv(self) -> bool:
|
4122
|
-
return self.prefix != self.base_prefix
|
4341
|
+
#
|
4123
4342
|
|
4343
|
+
class Request(Message, abc.ABC): # noqa
|
4344
|
+
pass
|
4124
4345
|
|
4125
|
-
|
4346
|
+
@dc.dataclass(frozen=True)
|
4347
|
+
class CommandRequest(Request):
|
4348
|
+
seq: int
|
4349
|
+
cmd: Command
|
4126
4350
|
|
4127
|
-
|
4128
|
-
|
4351
|
+
@dc.dataclass(frozen=True)
|
4352
|
+
class PingRequest(Request):
|
4353
|
+
time: float
|
4129
4354
|
|
4130
|
-
|
4355
|
+
#
|
4131
4356
|
|
4132
|
-
|
4133
|
-
|
4134
|
-
version_str=__import__('sys').version,
|
4135
|
-
prefix=__import__('sys').prefix,
|
4136
|
-
base_prefix=__import__('sys').base_prefix,
|
4137
|
-
config_vars=__import__('sysconfig').get_config_vars(),
|
4138
|
-
))"""
|
4357
|
+
class Response(Message, abc.ABC): # noqa
|
4358
|
+
pass
|
4139
4359
|
|
4140
|
-
|
4360
|
+
@dc.dataclass(frozen=True)
|
4361
|
+
class LogResponse(Response):
|
4362
|
+
s: str
|
4141
4363
|
|
4142
|
-
@
|
4143
|
-
|
4144
|
-
|
4145
|
-
|
4146
|
-
) -> InterpInspection:
|
4147
|
-
dct = json.loads(output)
|
4364
|
+
@dc.dataclass(frozen=True)
|
4365
|
+
class CommandResponse(Response):
|
4366
|
+
seq: int
|
4367
|
+
res: CommandOutputOrExceptionData
|
4148
4368
|
|
4149
|
-
|
4369
|
+
@dc.dataclass(frozen=True)
|
4370
|
+
class PingResponse(Response):
|
4371
|
+
time: float
|
4150
4372
|
|
4151
|
-
return InterpInspection(
|
4152
|
-
exe=exe,
|
4153
|
-
version=version,
|
4154
|
-
**{k: dct[k] for k in (
|
4155
|
-
'version_str',
|
4156
|
-
'prefix',
|
4157
|
-
'base_prefix',
|
4158
|
-
'config_vars',
|
4159
|
-
)},
|
4160
|
-
)
|
4161
4373
|
|
4162
|
-
|
4163
|
-
def running(cls) -> 'InterpInspection':
|
4164
|
-
return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
|
4374
|
+
##
|
4165
4375
|
|
4166
|
-
def _inspect(self, exe: str) -> InterpInspection:
|
4167
|
-
output = subprocess_check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
|
4168
|
-
return self._build_inspection(exe, output.decode())
|
4169
4376
|
|
4170
|
-
|
4171
|
-
|
4172
|
-
|
4173
|
-
|
4174
|
-
|
4175
|
-
|
4176
|
-
|
4177
|
-
except Exception as e: # noqa
|
4178
|
-
if log.isEnabledFor(logging.DEBUG):
|
4179
|
-
log.exception('Failed to inspect interp: %s', exe)
|
4180
|
-
ret = None
|
4181
|
-
self._cache[exe] = ret
|
4182
|
-
return ret
|
4377
|
+
class _RemoteLogHandler(logging.Handler):
|
4378
|
+
def __init__(
|
4379
|
+
self,
|
4380
|
+
chan: RemoteChannel,
|
4381
|
+
loop: ta.Any = None,
|
4382
|
+
) -> None:
|
4383
|
+
super().__init__()
|
4183
4384
|
|
4385
|
+
self._chan = chan
|
4386
|
+
self._loop = loop
|
4184
4387
|
|
4185
|
-
|
4388
|
+
def emit(self, record):
|
4389
|
+
msg = self.format(record)
|
4186
4390
|
|
4391
|
+
async def inner():
|
4392
|
+
await _RemoteProtocol.LogResponse(msg).send(self._chan)
|
4187
4393
|
|
4188
|
-
|
4189
|
-
|
4394
|
+
loop = self._loop
|
4395
|
+
if loop is None:
|
4396
|
+
loop = asyncio.get_running_loop()
|
4397
|
+
if loop is not None:
|
4398
|
+
asyncio.run_coroutine_threadsafe(inner(), loop)
|
4190
4399
|
|
4191
4400
|
|
4192
4401
|
##
|
4193
4402
|
|
4194
4403
|
|
4195
|
-
|
4196
|
-
|
4197
|
-
|
4404
|
+
class _RemoteCommandHandler:
|
4405
|
+
def __init__(
|
4406
|
+
self,
|
4407
|
+
chan: RemoteChannel,
|
4408
|
+
executor: CommandExecutor,
|
4409
|
+
*,
|
4410
|
+
stop: ta.Optional[asyncio.Event] = None,
|
4411
|
+
) -> None:
|
4412
|
+
super().__init__()
|
4198
4413
|
|
4199
|
-
|
4200
|
-
|
4201
|
-
|
4414
|
+
self._chan = chan
|
4415
|
+
self._executor = executor
|
4416
|
+
self._stop = stop if stop is not None else asyncio.Event()
|
4202
4417
|
|
4203
|
-
|
4204
|
-
|
4418
|
+
self._cmds_by_seq: ta.Dict[int, _RemoteCommandHandler._Command] = {}
|
4419
|
+
|
4420
|
+
@dc.dataclass(frozen=True)
|
4421
|
+
class _Command:
|
4422
|
+
req: _RemoteProtocol.CommandRequest
|
4423
|
+
fut: asyncio.Future
|
4424
|
+
|
4425
|
+
async def run(self) -> None:
|
4426
|
+
stop_task = asyncio.create_task(self._stop.wait())
|
4427
|
+
recv_task: ta.Optional[asyncio.Task] = None
|
4428
|
+
|
4429
|
+
while not self._stop.is_set():
|
4430
|
+
if recv_task is None:
|
4431
|
+
recv_task = asyncio.create_task(_RemoteProtocol.Request.recv(self._chan))
|
4432
|
+
|
4433
|
+
done, pending = await asyncio.wait([
|
4434
|
+
stop_task,
|
4435
|
+
recv_task,
|
4436
|
+
], return_when=asyncio.FIRST_COMPLETED)
|
4437
|
+
|
4438
|
+
if recv_task in done:
|
4439
|
+
msg: ta.Optional[_RemoteProtocol.Message] = check_isinstance(
|
4440
|
+
recv_task.result(),
|
4441
|
+
(_RemoteProtocol.Message, type(None)),
|
4442
|
+
)
|
4443
|
+
recv_task = None
|
4444
|
+
|
4445
|
+
if msg is None:
|
4446
|
+
break
|
4447
|
+
|
4448
|
+
await self._handle_message(msg)
|
4449
|
+
|
4450
|
+
async def _handle_message(self, msg: _RemoteProtocol.Message) -> None:
|
4451
|
+
if isinstance(msg, _RemoteProtocol.PingRequest):
|
4452
|
+
log.debug('Ping: %r', msg)
|
4453
|
+
await _RemoteProtocol.PingResponse(
|
4454
|
+
time=msg.time,
|
4455
|
+
).send(self._chan)
|
4456
|
+
|
4457
|
+
elif isinstance(msg, _RemoteProtocol.CommandRequest):
|
4458
|
+
fut = asyncio.create_task(self._handle_command_request(msg))
|
4459
|
+
self._cmds_by_seq[msg.seq] = _RemoteCommandHandler._Command(
|
4460
|
+
req=msg,
|
4461
|
+
fut=fut,
|
4462
|
+
)
|
4463
|
+
|
4464
|
+
else:
|
4465
|
+
raise TypeError(msg)
|
4466
|
+
|
4467
|
+
async def _handle_command_request(self, req: _RemoteProtocol.CommandRequest) -> None:
|
4468
|
+
res = await self._executor.try_execute(
|
4469
|
+
req.cmd,
|
4470
|
+
log=log,
|
4471
|
+
omit_exc_object=True,
|
4472
|
+
)
|
4473
|
+
|
4474
|
+
await _RemoteProtocol.CommandResponse(
|
4475
|
+
seq=req.seq,
|
4476
|
+
res=CommandOutputOrExceptionData(
|
4477
|
+
output=res.output,
|
4478
|
+
exception=res.exception,
|
4479
|
+
),
|
4480
|
+
).send(self._chan)
|
4481
|
+
|
4482
|
+
self._cmds_by_seq.pop(req.seq) # noqa
|
4483
|
+
|
4484
|
+
|
4485
|
+
##
|
4486
|
+
|
4487
|
+
|
4488
|
+
@dc.dataclass()
|
4489
|
+
class RemoteCommandError(Exception):
|
4490
|
+
e: CommandException
|
4491
|
+
|
4492
|
+
|
4493
|
+
class RemoteCommandExecutor(CommandExecutor):
|
4494
|
+
def __init__(self, chan: RemoteChannel) -> None:
|
4495
|
+
super().__init__()
|
4496
|
+
|
4497
|
+
self._chan = chan
|
4498
|
+
|
4499
|
+
self._cmd_seq = itertools.count()
|
4500
|
+
self._queue: asyncio.Queue = asyncio.Queue() # asyncio.Queue[RemoteCommandExecutor._Request]
|
4501
|
+
self._stop = asyncio.Event()
|
4502
|
+
self._loop_task: ta.Optional[asyncio.Task] = None
|
4503
|
+
self._reqs_by_seq: ta.Dict[int, RemoteCommandExecutor._Request] = {}
|
4504
|
+
|
4505
|
+
#
|
4506
|
+
|
4507
|
+
async def start(self) -> None:
|
4508
|
+
check_none(self._loop_task)
|
4509
|
+
check_state(not self._stop.is_set())
|
4510
|
+
self._loop_task = asyncio.create_task(self._loop())
|
4511
|
+
|
4512
|
+
async def aclose(self) -> None:
|
4513
|
+
self._stop.set()
|
4514
|
+
if self._loop_task is not None:
|
4515
|
+
await self._loop_task
|
4516
|
+
|
4517
|
+
#
|
4518
|
+
|
4519
|
+
@dc.dataclass(frozen=True)
|
4520
|
+
class _Request:
|
4521
|
+
seq: int
|
4522
|
+
cmd: Command
|
4523
|
+
fut: asyncio.Future
|
4524
|
+
|
4525
|
+
async def _loop(self) -> None:
|
4526
|
+
log.debug('RemoteCommandExecutor loop start: %r', self)
|
4527
|
+
|
4528
|
+
stop_task = asyncio.create_task(self._stop.wait())
|
4529
|
+
queue_task: ta.Optional[asyncio.Task] = None
|
4530
|
+
recv_task: ta.Optional[asyncio.Task] = None
|
4531
|
+
|
4532
|
+
while not self._stop.is_set():
|
4533
|
+
if queue_task is None:
|
4534
|
+
queue_task = asyncio.create_task(self._queue.get())
|
4535
|
+
if recv_task is None:
|
4536
|
+
recv_task = asyncio.create_task(_RemoteProtocol.Message.recv(self._chan))
|
4537
|
+
|
4538
|
+
done, pending = await asyncio.wait([
|
4539
|
+
stop_task,
|
4540
|
+
queue_task,
|
4541
|
+
recv_task,
|
4542
|
+
], return_when=asyncio.FIRST_COMPLETED)
|
4543
|
+
|
4544
|
+
if queue_task in done:
|
4545
|
+
req = check_isinstance(queue_task.result(), RemoteCommandExecutor._Request)
|
4546
|
+
queue_task = None
|
4547
|
+
await self._handle_request(req)
|
4548
|
+
|
4549
|
+
if recv_task in done:
|
4550
|
+
msg: ta.Optional[_RemoteProtocol.Message] = check_isinstance(
|
4551
|
+
recv_task.result(),
|
4552
|
+
(_RemoteProtocol.Message, type(None)),
|
4553
|
+
)
|
4554
|
+
recv_task = None
|
4555
|
+
|
4556
|
+
if msg is None:
|
4557
|
+
log.debug('RemoteCommandExecutor got eof: %r', self)
|
4558
|
+
break
|
4559
|
+
|
4560
|
+
await self._handle_message(msg)
|
4561
|
+
|
4562
|
+
log.debug('RemoteCommandExecutor loop stopping: %r', self)
|
4563
|
+
|
4564
|
+
for task in [
|
4565
|
+
stop_task,
|
4566
|
+
queue_task,
|
4567
|
+
recv_task,
|
4568
|
+
]:
|
4569
|
+
if task is not None and not task.done():
|
4570
|
+
task.cancel()
|
4571
|
+
|
4572
|
+
for req in self._reqs_by_seq.values():
|
4573
|
+
req.fut.cancel()
|
4574
|
+
|
4575
|
+
log.debug('RemoteCommandExecutor loop exited: %r', self)
|
4576
|
+
|
4577
|
+
async def _handle_request(self, req: _Request) -> None:
|
4578
|
+
self._reqs_by_seq[req.seq] = req
|
4579
|
+
await _RemoteProtocol.CommandRequest(
|
4580
|
+
seq=req.seq,
|
4581
|
+
cmd=req.cmd,
|
4582
|
+
).send(self._chan)
|
4583
|
+
|
4584
|
+
async def _handle_message(self, msg: _RemoteProtocol.Message) -> None:
|
4585
|
+
if isinstance(msg, _RemoteProtocol.PingRequest):
|
4586
|
+
log.debug('Ping: %r', msg)
|
4587
|
+
await _RemoteProtocol.PingResponse(
|
4588
|
+
time=msg.time,
|
4589
|
+
).send(self._chan)
|
4590
|
+
|
4591
|
+
elif isinstance(msg, _RemoteProtocol.LogResponse):
|
4592
|
+
log.info(msg.s)
|
4593
|
+
|
4594
|
+
elif isinstance(msg, _RemoteProtocol.CommandResponse):
|
4595
|
+
req = self._reqs_by_seq.pop(msg.seq)
|
4596
|
+
req.fut.set_result(msg.res)
|
4597
|
+
|
4598
|
+
else:
|
4599
|
+
raise TypeError(msg)
|
4600
|
+
|
4601
|
+
#
|
4602
|
+
|
4603
|
+
async def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
|
4604
|
+
req = RemoteCommandExecutor._Request(
|
4605
|
+
seq=next(self._cmd_seq),
|
4606
|
+
cmd=cmd,
|
4607
|
+
fut=asyncio.Future(),
|
4608
|
+
)
|
4609
|
+
await self._queue.put(req)
|
4610
|
+
return await req.fut
|
4611
|
+
|
4612
|
+
# @ta.override
|
4613
|
+
async def execute(self, cmd: Command) -> Command.Output:
|
4614
|
+
r = await self._remote_execute(cmd)
|
4615
|
+
if (e := r.exception) is not None:
|
4616
|
+
raise RemoteCommandError(e)
|
4617
|
+
else:
|
4618
|
+
return check_not_none(r.output)
|
4619
|
+
|
4620
|
+
# @ta.override
|
4621
|
+
async def try_execute(
|
4622
|
+
self,
|
4623
|
+
cmd: Command,
|
4624
|
+
*,
|
4625
|
+
log: ta.Optional[logging.Logger] = None,
|
4626
|
+
omit_exc_object: bool = False,
|
4627
|
+
) -> CommandOutputOrException:
|
4628
|
+
try:
|
4629
|
+
r = await self._remote_execute(cmd)
|
4630
|
+
|
4631
|
+
except Exception as e: # noqa
|
4632
|
+
if log is not None:
|
4633
|
+
log.exception('Exception executing remote command: %r', type(cmd))
|
4634
|
+
|
4635
|
+
return CommandOutputOrExceptionData(exception=CommandException.of(
|
4636
|
+
e,
|
4637
|
+
omit_exc_object=omit_exc_object,
|
4638
|
+
cmd=cmd,
|
4639
|
+
))
|
4640
|
+
|
4641
|
+
else:
|
4642
|
+
return r
|
4643
|
+
|
4644
|
+
|
4645
|
+
########################################
|
4646
|
+
# ../../../omlish/lite/asyncio/subprocesses.py
|
4647
|
+
|
4648
|
+
|
4649
|
+
##
|
4650
|
+
|
4651
|
+
|
4652
|
+
@contextlib.asynccontextmanager
|
4653
|
+
async def asyncio_subprocess_popen(
|
4654
|
+
*cmd: str,
|
4655
|
+
shell: bool = False,
|
4656
|
+
timeout: ta.Optional[float] = None,
|
4657
|
+
**kwargs: ta.Any,
|
4658
|
+
) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
|
4659
|
+
fac: ta.Any
|
4660
|
+
if shell:
|
4661
|
+
fac = functools.partial(
|
4662
|
+
asyncio.create_subprocess_shell,
|
4663
|
+
check_single(cmd),
|
4664
|
+
)
|
4665
|
+
else:
|
4666
|
+
fac = functools.partial(
|
4667
|
+
asyncio.create_subprocess_exec,
|
4668
|
+
*cmd,
|
4669
|
+
)
|
4670
|
+
|
4671
|
+
with subprocess_common_context(
|
4672
|
+
*cmd,
|
4673
|
+
shell=shell,
|
4674
|
+
timeout=timeout,
|
4675
|
+
**kwargs,
|
4676
|
+
):
|
4677
|
+
proc: asyncio.subprocess.Process
|
4678
|
+
proc = await fac(**kwargs)
|
4679
|
+
try:
|
4680
|
+
yield proc
|
4681
|
+
|
4682
|
+
finally:
|
4683
|
+
await asyncio_maybe_timeout(proc.wait(), timeout)
|
4684
|
+
|
4685
|
+
|
4686
|
+
##
|
4687
|
+
|
4688
|
+
|
4689
|
+
class AsyncioProcessCommunicator:
|
4690
|
+
def __init__(
|
4691
|
+
self,
|
4692
|
+
proc: asyncio.subprocess.Process,
|
4693
|
+
loop: ta.Optional[ta.Any] = None,
|
4694
|
+
) -> None:
|
4695
|
+
super().__init__()
|
4696
|
+
|
4697
|
+
if loop is None:
|
4698
|
+
loop = asyncio.get_running_loop()
|
4699
|
+
|
4700
|
+
self._proc = proc
|
4701
|
+
self._loop = loop
|
4702
|
+
|
4703
|
+
self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check_isinstance(
|
4704
|
+
proc._transport, # type: ignore # noqa
|
4705
|
+
asyncio.base_subprocess.BaseSubprocessTransport,
|
4706
|
+
)
|
4707
|
+
|
4708
|
+
@property
|
4709
|
+
def _debug(self) -> bool:
|
4710
|
+
return self._loop.get_debug()
|
4711
|
+
|
4712
|
+
async def _feed_stdin(self, input: bytes) -> None: # noqa
|
4713
|
+
stdin = check_not_none(self._proc.stdin)
|
4714
|
+
try:
|
4715
|
+
if input is not None:
|
4716
|
+
stdin.write(input)
|
4717
|
+
if self._debug:
|
4718
|
+
log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
|
4719
|
+
|
4720
|
+
await stdin.drain()
|
4721
|
+
|
4722
|
+
except (BrokenPipeError, ConnectionResetError) as exc:
|
4723
|
+
# communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
|
4724
|
+
# exceptions.
|
4725
|
+
if self._debug:
|
4726
|
+
log.debug('%r communicate: stdin got %r', self, exc)
|
4727
|
+
|
4728
|
+
if self._debug:
|
4729
|
+
log.debug('%r communicate: close stdin', self)
|
4730
|
+
|
4731
|
+
stdin.close()
|
4732
|
+
|
4733
|
+
async def _noop(self) -> None:
|
4734
|
+
return None
|
4735
|
+
|
4736
|
+
async def _read_stream(self, fd: int) -> bytes:
|
4737
|
+
transport: ta.Any = check_not_none(self._transport.get_pipe_transport(fd))
|
4738
|
+
|
4739
|
+
if fd == 2:
|
4740
|
+
stream = check_not_none(self._proc.stderr)
|
4741
|
+
else:
|
4742
|
+
check_equal(fd, 1)
|
4743
|
+
stream = check_not_none(self._proc.stdout)
|
4744
|
+
|
4745
|
+
if self._debug:
|
4746
|
+
name = 'stdout' if fd == 1 else 'stderr'
|
4747
|
+
log.debug('%r communicate: read %s', self, name)
|
4748
|
+
|
4749
|
+
output = await stream.read()
|
4750
|
+
|
4751
|
+
if self._debug:
|
4752
|
+
name = 'stdout' if fd == 1 else 'stderr'
|
4753
|
+
log.debug('%r communicate: close %s', self, name)
|
4754
|
+
|
4755
|
+
transport.close()
|
4756
|
+
|
4757
|
+
return output
|
4758
|
+
|
4759
|
+
class Communication(ta.NamedTuple):
|
4760
|
+
stdout: ta.Optional[bytes]
|
4761
|
+
stderr: ta.Optional[bytes]
|
4762
|
+
|
4763
|
+
async def _communicate(
|
4764
|
+
self,
|
4765
|
+
input: ta.Any = None, # noqa
|
4766
|
+
) -> Communication:
|
4767
|
+
stdin_fut: ta.Any
|
4768
|
+
if self._proc.stdin is not None:
|
4769
|
+
stdin_fut = self._feed_stdin(input)
|
4770
|
+
else:
|
4771
|
+
stdin_fut = self._noop()
|
4772
|
+
|
4773
|
+
stdout_fut: ta.Any
|
4774
|
+
if self._proc.stdout is not None:
|
4775
|
+
stdout_fut = self._read_stream(1)
|
4776
|
+
else:
|
4777
|
+
stdout_fut = self._noop()
|
4778
|
+
|
4779
|
+
stderr_fut: ta.Any
|
4780
|
+
if self._proc.stderr is not None:
|
4781
|
+
stderr_fut = self._read_stream(2)
|
4782
|
+
else:
|
4783
|
+
stderr_fut = self._noop()
|
4784
|
+
|
4785
|
+
stdin_res, stdout_res, stderr_res = await asyncio.gather(stdin_fut, stdout_fut, stderr_fut)
|
4786
|
+
|
4787
|
+
await self._proc.wait()
|
4788
|
+
|
4789
|
+
return AsyncioProcessCommunicator.Communication(stdout_res, stderr_res)
|
4790
|
+
|
4791
|
+
async def communicate(
|
4792
|
+
self,
|
4793
|
+
input: ta.Any = None, # noqa
|
4794
|
+
timeout: ta.Optional[float] = None,
|
4795
|
+
) -> Communication:
|
4796
|
+
return await asyncio_maybe_timeout(self._communicate(input), timeout)
|
4797
|
+
|
4798
|
+
|
4799
|
+
async def asyncio_subprocess_communicate(
|
4800
|
+
proc: asyncio.subprocess.Process,
|
4801
|
+
input: ta.Any = None, # noqa
|
4802
|
+
timeout: ta.Optional[float] = None,
|
4803
|
+
) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
|
4804
|
+
return await AsyncioProcessCommunicator(proc).communicate(input, timeout) # noqa
|
4805
|
+
|
4806
|
+
|
4807
|
+
##
|
4808
|
+
|
4809
|
+
|
4810
|
+
async def _asyncio_subprocess_check_run(
|
4811
|
+
*args: str,
|
4812
|
+
input: ta.Any = None, # noqa
|
4813
|
+
timeout: ta.Optional[float] = None,
|
4814
|
+
**kwargs: ta.Any,
|
4815
|
+
) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
|
4816
|
+
args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
|
4817
|
+
|
4818
|
+
proc: asyncio.subprocess.Process
|
4819
|
+
async with asyncio_subprocess_popen(*args, **kwargs) as proc:
|
4820
|
+
stdout, stderr = await asyncio_subprocess_communicate(proc, input, timeout)
|
4821
|
+
|
4822
|
+
if proc.returncode:
|
4823
|
+
raise subprocess.CalledProcessError(
|
4824
|
+
proc.returncode,
|
4825
|
+
args,
|
4826
|
+
output=stdout,
|
4827
|
+
stderr=stderr,
|
4828
|
+
)
|
4829
|
+
|
4830
|
+
return stdout, stderr
|
4831
|
+
|
4832
|
+
|
4833
|
+
async def asyncio_subprocess_check_call(
|
4834
|
+
*args: str,
|
4835
|
+
stdout: ta.Any = sys.stderr,
|
4836
|
+
input: ta.Any = None, # noqa
|
4837
|
+
timeout: ta.Optional[float] = None,
|
4838
|
+
**kwargs: ta.Any,
|
4839
|
+
) -> None:
|
4840
|
+
_, _ = await _asyncio_subprocess_check_run(
|
4841
|
+
*args,
|
4842
|
+
stdout=stdout,
|
4843
|
+
input=input,
|
4844
|
+
timeout=timeout,
|
4845
|
+
**kwargs,
|
4846
|
+
)
|
4847
|
+
|
4848
|
+
|
4849
|
+
async def asyncio_subprocess_check_output(
|
4850
|
+
*args: str,
|
4851
|
+
input: ta.Any = None, # noqa
|
4852
|
+
timeout: ta.Optional[float] = None,
|
4853
|
+
**kwargs: ta.Any,
|
4854
|
+
) -> bytes:
|
4855
|
+
stdout, stderr = await _asyncio_subprocess_check_run(
|
4856
|
+
*args,
|
4857
|
+
stdout=asyncio.subprocess.PIPE,
|
4858
|
+
input=input,
|
4859
|
+
timeout=timeout,
|
4860
|
+
**kwargs,
|
4861
|
+
)
|
4862
|
+
|
4863
|
+
return check_not_none(stdout)
|
4864
|
+
|
4865
|
+
|
4866
|
+
async def asyncio_subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
|
4867
|
+
return (await asyncio_subprocess_check_output(*args, **kwargs)).decode().strip()
|
4868
|
+
|
4869
|
+
|
4870
|
+
##
|
4871
|
+
|
4872
|
+
|
4873
|
+
async def _asyncio_subprocess_try_run(
|
4874
|
+
fn: ta.Callable[..., ta.Awaitable[T]],
|
4875
|
+
*args: ta.Any,
|
4876
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4877
|
+
**kwargs: ta.Any,
|
4878
|
+
) -> ta.Union[T, Exception]:
|
4879
|
+
try:
|
4880
|
+
return await fn(*args, **kwargs)
|
4881
|
+
except try_exceptions as e: # noqa
|
4882
|
+
if log.isEnabledFor(logging.DEBUG):
|
4883
|
+
log.exception('command failed')
|
4884
|
+
return e
|
4885
|
+
|
4886
|
+
|
4887
|
+
async def asyncio_subprocess_try_call(
|
4888
|
+
*args: str,
|
4889
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4890
|
+
**kwargs: ta.Any,
|
4891
|
+
) -> bool:
|
4892
|
+
if isinstance(await _asyncio_subprocess_try_run(
|
4893
|
+
asyncio_subprocess_check_call,
|
4894
|
+
*args,
|
4895
|
+
try_exceptions=try_exceptions,
|
4896
|
+
**kwargs,
|
4897
|
+
), Exception):
|
4898
|
+
return False
|
4899
|
+
else:
|
4900
|
+
return True
|
4901
|
+
|
4902
|
+
|
4903
|
+
async def asyncio_subprocess_try_output(
|
4904
|
+
*args: str,
|
4905
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
4906
|
+
**kwargs: ta.Any,
|
4907
|
+
) -> ta.Optional[bytes]:
|
4908
|
+
if isinstance(ret := await _asyncio_subprocess_try_run(
|
4909
|
+
asyncio_subprocess_check_output,
|
4910
|
+
*args,
|
4911
|
+
try_exceptions=try_exceptions,
|
4912
|
+
**kwargs,
|
4913
|
+
), Exception):
|
4914
|
+
return None
|
4915
|
+
else:
|
4916
|
+
return ret
|
4917
|
+
|
4918
|
+
|
4919
|
+
async def asyncio_subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|
4920
|
+
out = await asyncio_subprocess_try_output(*args, **kwargs)
|
4921
|
+
return out.decode().strip() if out is not None else None
|
4922
|
+
|
4923
|
+
|
4924
|
+
########################################
|
4925
|
+
# ../../../omdev/interp/inspect.py
|
4926
|
+
|
4927
|
+
|
4928
|
+
@dc.dataclass(frozen=True)
|
4929
|
+
class InterpInspection:
|
4930
|
+
exe: str
|
4931
|
+
version: Version
|
4932
|
+
|
4933
|
+
version_str: str
|
4934
|
+
config_vars: ta.Mapping[str, str]
|
4935
|
+
prefix: str
|
4936
|
+
base_prefix: str
|
4937
|
+
|
4938
|
+
@property
|
4939
|
+
def opts(self) -> InterpOpts:
|
4940
|
+
return InterpOpts(
|
4941
|
+
threaded=bool(self.config_vars.get('Py_GIL_DISABLED')),
|
4942
|
+
debug=bool(self.config_vars.get('Py_DEBUG')),
|
4943
|
+
)
|
4944
|
+
|
4945
|
+
@property
|
4946
|
+
def iv(self) -> InterpVersion:
|
4947
|
+
return InterpVersion(
|
4948
|
+
version=self.version,
|
4949
|
+
opts=self.opts,
|
4950
|
+
)
|
4951
|
+
|
4952
|
+
@property
|
4953
|
+
def is_venv(self) -> bool:
|
4954
|
+
return self.prefix != self.base_prefix
|
4955
|
+
|
4956
|
+
|
4957
|
+
class InterpInspector:
|
4958
|
+
def __init__(self) -> None:
|
4959
|
+
super().__init__()
|
4960
|
+
|
4961
|
+
self._cache: ta.Dict[str, ta.Optional[InterpInspection]] = {}
|
4962
|
+
|
4963
|
+
_RAW_INSPECTION_CODE = """
|
4964
|
+
__import__('json').dumps(dict(
|
4965
|
+
version_str=__import__('sys').version,
|
4966
|
+
prefix=__import__('sys').prefix,
|
4967
|
+
base_prefix=__import__('sys').base_prefix,
|
4968
|
+
config_vars=__import__('sysconfig').get_config_vars(),
|
4969
|
+
))"""
|
4970
|
+
|
4971
|
+
_INSPECTION_CODE = ''.join(l.strip() for l in _RAW_INSPECTION_CODE.splitlines())
|
4972
|
+
|
4973
|
+
@staticmethod
|
4974
|
+
def _build_inspection(
|
4975
|
+
exe: str,
|
4976
|
+
output: str,
|
4977
|
+
) -> InterpInspection:
|
4978
|
+
dct = json.loads(output)
|
4979
|
+
|
4980
|
+
version = Version(dct['version_str'].split()[0])
|
4981
|
+
|
4982
|
+
return InterpInspection(
|
4983
|
+
exe=exe,
|
4984
|
+
version=version,
|
4985
|
+
**{k: dct[k] for k in (
|
4986
|
+
'version_str',
|
4987
|
+
'prefix',
|
4988
|
+
'base_prefix',
|
4989
|
+
'config_vars',
|
4990
|
+
)},
|
4991
|
+
)
|
4992
|
+
|
4993
|
+
@classmethod
|
4994
|
+
def running(cls) -> 'InterpInspection':
|
4995
|
+
return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
|
4996
|
+
|
4997
|
+
async def _inspect(self, exe: str) -> InterpInspection:
|
4998
|
+
output = await asyncio_subprocess_check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
|
4999
|
+
return self._build_inspection(exe, output.decode())
|
5000
|
+
|
5001
|
+
async def inspect(self, exe: str) -> ta.Optional[InterpInspection]:
|
5002
|
+
try:
|
5003
|
+
return self._cache[exe]
|
5004
|
+
except KeyError:
|
5005
|
+
ret: ta.Optional[InterpInspection]
|
5006
|
+
try:
|
5007
|
+
ret = await self._inspect(exe)
|
5008
|
+
except Exception as e: # noqa
|
5009
|
+
if log.isEnabledFor(logging.DEBUG):
|
5010
|
+
log.exception('Failed to inspect interp: %s', exe)
|
5011
|
+
ret = None
|
5012
|
+
self._cache[exe] = ret
|
5013
|
+
return ret
|
5014
|
+
|
5015
|
+
|
5016
|
+
INTERP_INSPECTOR = InterpInspector()
|
5017
|
+
|
5018
|
+
|
5019
|
+
########################################
|
5020
|
+
# ../commands/subprocess.py
|
5021
|
+
|
5022
|
+
|
5023
|
+
##
|
5024
|
+
|
5025
|
+
|
5026
|
+
@dc.dataclass(frozen=True)
|
5027
|
+
class SubprocessCommand(Command['SubprocessCommand.Output']):
|
5028
|
+
cmd: ta.Sequence[str]
|
5029
|
+
|
5030
|
+
shell: bool = False
|
5031
|
+
cwd: ta.Optional[str] = None
|
5032
|
+
env: ta.Optional[ta.Mapping[str, str]] = None
|
5033
|
+
|
5034
|
+
stdout: str = 'pipe' # SubprocessChannelOption
|
5035
|
+
stderr: str = 'pipe' # SubprocessChannelOption
|
4205
5036
|
|
4206
5037
|
input: ta.Optional[bytes] = None
|
4207
5038
|
timeout: ta.Optional[float] = None
|
4208
5039
|
|
4209
|
-
def __post_init__(self) -> None:
|
4210
|
-
check_not_isinstance(self.cmd, str)
|
5040
|
+
def __post_init__(self) -> None:
|
5041
|
+
check_not_isinstance(self.cmd, str)
|
5042
|
+
|
5043
|
+
@dc.dataclass(frozen=True)
|
5044
|
+
class Output(Command.Output):
|
5045
|
+
rc: int
|
5046
|
+
pid: int
|
5047
|
+
|
5048
|
+
elapsed_s: float
|
5049
|
+
|
5050
|
+
stdout: ta.Optional[bytes] = None
|
5051
|
+
stderr: ta.Optional[bytes] = None
|
5052
|
+
|
5053
|
+
|
5054
|
+
##
|
5055
|
+
|
5056
|
+
|
5057
|
+
class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
|
5058
|
+
async def execute(self, cmd: SubprocessCommand) -> SubprocessCommand.Output:
|
5059
|
+
proc: asyncio.subprocess.Process
|
5060
|
+
async with asyncio_subprocess_popen(
|
5061
|
+
*subprocess_maybe_shell_wrap_exec(*cmd.cmd),
|
5062
|
+
|
5063
|
+
shell=cmd.shell,
|
5064
|
+
cwd=cmd.cwd,
|
5065
|
+
env={**os.environ, **(cmd.env or {})},
|
5066
|
+
|
5067
|
+
stdin=subprocess.PIPE if cmd.input is not None else None,
|
5068
|
+
stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, cmd.stdout)],
|
5069
|
+
stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, cmd.stderr)],
|
5070
|
+
|
5071
|
+
timeout=cmd.timeout,
|
5072
|
+
) as proc:
|
5073
|
+
start_time = time.time()
|
5074
|
+
stdout, stderr = await asyncio_subprocess_communicate(
|
5075
|
+
proc,
|
5076
|
+
input=cmd.input,
|
5077
|
+
timeout=cmd.timeout,
|
5078
|
+
)
|
5079
|
+
end_time = time.time()
|
5080
|
+
|
5081
|
+
return SubprocessCommand.Output(
|
5082
|
+
rc=check_not_none(proc.returncode),
|
5083
|
+
pid=proc.pid,
|
5084
|
+
|
5085
|
+
elapsed_s=end_time - start_time,
|
5086
|
+
|
5087
|
+
stdout=stdout, # noqa
|
5088
|
+
stderr=stderr, # noqa
|
5089
|
+
)
|
5090
|
+
|
5091
|
+
|
5092
|
+
########################################
|
5093
|
+
# ../remote/_main.py
|
5094
|
+
|
5095
|
+
|
5096
|
+
##
|
5097
|
+
|
5098
|
+
|
5099
|
+
class _RemoteExecutionLogHandler(logging.Handler):
|
5100
|
+
def __init__(self, fn: ta.Callable[[str], None]) -> None:
|
5101
|
+
super().__init__()
|
5102
|
+
self._fn = fn
|
5103
|
+
|
5104
|
+
def emit(self, record):
|
5105
|
+
msg = self.format(record)
|
5106
|
+
self._fn(msg)
|
5107
|
+
|
5108
|
+
|
5109
|
+
##
|
5110
|
+
|
5111
|
+
|
5112
|
+
class _RemoteExecutionMain:
|
5113
|
+
def __init__(
|
5114
|
+
self,
|
5115
|
+
chan: RemoteChannel,
|
5116
|
+
) -> None:
|
5117
|
+
super().__init__()
|
5118
|
+
|
5119
|
+
self._chan = chan
|
5120
|
+
|
5121
|
+
self.__bootstrap: ta.Optional[MainBootstrap] = None
|
5122
|
+
self.__injector: ta.Optional[Injector] = None
|
5123
|
+
|
5124
|
+
@property
|
5125
|
+
def _bootstrap(self) -> MainBootstrap:
|
5126
|
+
return check_not_none(self.__bootstrap)
|
5127
|
+
|
5128
|
+
@property
|
5129
|
+
def _injector(self) -> Injector:
|
5130
|
+
return check_not_none(self.__injector)
|
5131
|
+
|
5132
|
+
#
|
5133
|
+
|
5134
|
+
def _timebomb_main(
|
5135
|
+
self,
|
5136
|
+
delay_s: float,
|
5137
|
+
*,
|
5138
|
+
sig: int = signal.SIGINT,
|
5139
|
+
code: int = 1,
|
5140
|
+
) -> None:
|
5141
|
+
time.sleep(delay_s)
|
5142
|
+
|
5143
|
+
if (pgid := os.getpgid(0)) == os.getpid():
|
5144
|
+
os.killpg(pgid, sig)
|
5145
|
+
|
5146
|
+
os._exit(code) # noqa
|
5147
|
+
|
5148
|
+
@cached_nullary
|
5149
|
+
def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
|
5150
|
+
if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
|
5151
|
+
return None
|
5152
|
+
|
5153
|
+
thr = threading.Thread(
|
5154
|
+
target=functools.partial(self._timebomb_main, tbd),
|
5155
|
+
name=f'{self.__class__.__name__}.timebomb',
|
5156
|
+
daemon=True,
|
5157
|
+
)
|
5158
|
+
|
5159
|
+
thr.start()
|
4211
5160
|
|
4212
|
-
|
4213
|
-
class Output(Command.Output):
|
4214
|
-
rc: int
|
4215
|
-
pid: int
|
5161
|
+
log.debug('Started timebomb thread: %r', thr)
|
4216
5162
|
|
4217
|
-
|
5163
|
+
return thr
|
4218
5164
|
|
4219
|
-
|
4220
|
-
stderr: ta.Optional[bytes] = None
|
5165
|
+
#
|
4221
5166
|
|
5167
|
+
@cached_nullary
|
5168
|
+
def _log_handler(self) -> _RemoteLogHandler:
|
5169
|
+
return _RemoteLogHandler(self._chan)
|
4222
5170
|
|
4223
|
-
|
5171
|
+
#
|
4224
5172
|
|
5173
|
+
async def _setup(self) -> None:
|
5174
|
+
check_none(self.__bootstrap)
|
5175
|
+
check_none(self.__injector)
|
4225
5176
|
|
4226
|
-
|
4227
|
-
def execute(self, inp: SubprocessCommand) -> SubprocessCommand.Output:
|
4228
|
-
with subprocess.Popen(
|
4229
|
-
subprocess_maybe_shell_wrap_exec(*inp.cmd),
|
5177
|
+
# Bootstrap
|
4230
5178
|
|
4231
|
-
|
4232
|
-
cwd=inp.cwd,
|
4233
|
-
env={**os.environ, **(inp.env or {})},
|
5179
|
+
self.__bootstrap = check_not_none(await self._chan.recv_obj(MainBootstrap))
|
4234
5180
|
|
4235
|
-
|
4236
|
-
|
4237
|
-
stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stderr)],
|
4238
|
-
) as proc:
|
4239
|
-
start_time = time.time()
|
4240
|
-
stdout, stderr = proc.communicate(
|
4241
|
-
input=inp.input,
|
4242
|
-
timeout=inp.timeout,
|
4243
|
-
)
|
4244
|
-
end_time = time.time()
|
5181
|
+
if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
|
5182
|
+
pycharm_debug_connect(prd)
|
4245
5183
|
|
4246
|
-
|
4247
|
-
rc=proc.returncode,
|
4248
|
-
pid=proc.pid,
|
5184
|
+
self.__injector = main_bootstrap(self._bootstrap)
|
4249
5185
|
|
4250
|
-
|
5186
|
+
self._chan.set_marshaler(self._injector[ObjMarshalerManager])
|
4251
5187
|
|
4252
|
-
|
4253
|
-
|
5188
|
+
# Post-bootstrap
|
5189
|
+
|
5190
|
+
if self._bootstrap.remote_config.set_pgid:
|
5191
|
+
if os.getpgid(0) != os.getpid():
|
5192
|
+
log.debug('Setting pgid')
|
5193
|
+
os.setpgid(0, 0)
|
5194
|
+
|
5195
|
+
if (ds := self._bootstrap.remote_config.deathsig) is not None:
|
5196
|
+
log.debug('Setting deathsig: %s', ds)
|
5197
|
+
set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
|
5198
|
+
|
5199
|
+
self._timebomb_thread()
|
5200
|
+
|
5201
|
+
if self._bootstrap.remote_config.forward_logging:
|
5202
|
+
log.debug('Installing log forwarder')
|
5203
|
+
logging.root.addHandler(self._log_handler())
|
5204
|
+
|
5205
|
+
#
|
5206
|
+
|
5207
|
+
async def run(self) -> None:
|
5208
|
+
await self._setup()
|
5209
|
+
|
5210
|
+
executor = self._injector[LocalCommandExecutor]
|
5211
|
+
|
5212
|
+
handler = _RemoteCommandHandler(self._chan, executor)
|
5213
|
+
|
5214
|
+
await handler.run()
|
5215
|
+
|
5216
|
+
|
5217
|
+
def _remote_execution_main() -> None:
|
5218
|
+
rt = pyremote_bootstrap_finalize() # noqa
|
5219
|
+
|
5220
|
+
async def inner() -> None:
|
5221
|
+
input = await asyncio_open_stream_reader(rt.input) # noqa
|
5222
|
+
output = await asyncio_open_stream_writer(rt.output)
|
5223
|
+
|
5224
|
+
chan = RemoteChannelImpl(
|
5225
|
+
input,
|
5226
|
+
output,
|
4254
5227
|
)
|
4255
5228
|
|
5229
|
+
await _RemoteExecutionMain(chan).run()
|
5230
|
+
|
5231
|
+
asyncio.run(inner())
|
5232
|
+
|
4256
5233
|
|
4257
5234
|
########################################
|
4258
5235
|
# ../remote/spawning.py
|
4259
5236
|
|
4260
5237
|
|
4261
|
-
|
5238
|
+
##
|
5239
|
+
|
5240
|
+
|
5241
|
+
class RemoteSpawning(abc.ABC):
|
4262
5242
|
@dc.dataclass(frozen=True)
|
4263
5243
|
class Target:
|
4264
5244
|
shell: ta.Optional[str] = None
|
@@ -4269,15 +5249,35 @@ class RemoteSpawning:
|
|
4269
5249
|
|
4270
5250
|
stderr: ta.Optional[str] = None # SubprocessChannelOption
|
4271
5251
|
|
4272
|
-
|
5252
|
+
@dc.dataclass(frozen=True)
|
5253
|
+
class Spawned:
|
5254
|
+
stdin: asyncio.StreamWriter
|
5255
|
+
stdout: asyncio.StreamReader
|
5256
|
+
stderr: ta.Optional[asyncio.StreamReader]
|
5257
|
+
|
5258
|
+
@abc.abstractmethod
|
5259
|
+
def spawn(
|
5260
|
+
self,
|
5261
|
+
tgt: Target,
|
5262
|
+
src: str,
|
5263
|
+
*,
|
5264
|
+
timeout: ta.Optional[float] = None,
|
5265
|
+
debug: bool = False,
|
5266
|
+
) -> ta.AsyncContextManager[Spawned]:
|
5267
|
+
raise NotImplementedError
|
4273
5268
|
|
4274
|
-
|
5269
|
+
|
5270
|
+
##
|
5271
|
+
|
5272
|
+
|
5273
|
+
class SubprocessRemoteSpawning(RemoteSpawning):
|
5274
|
+
class _PreparedCmd(ta.NamedTuple): # noqa
|
4275
5275
|
cmd: ta.Sequence[str]
|
4276
5276
|
shell: bool
|
4277
5277
|
|
4278
5278
|
def _prepare_cmd(
|
4279
5279
|
self,
|
4280
|
-
tgt: Target,
|
5280
|
+
tgt: RemoteSpawning.Target,
|
4281
5281
|
src: str,
|
4282
5282
|
) -> _PreparedCmd:
|
4283
5283
|
if tgt.shell is not None:
|
@@ -4285,44 +5285,38 @@ class RemoteSpawning:
|
|
4285
5285
|
if tgt.shell_quote:
|
4286
5286
|
sh_src = shlex.quote(sh_src)
|
4287
5287
|
sh_cmd = f'{tgt.shell} {sh_src}'
|
4288
|
-
return
|
4289
|
-
cmd=[sh_cmd],
|
4290
|
-
shell=True,
|
4291
|
-
)
|
5288
|
+
return SubprocessRemoteSpawning._PreparedCmd([sh_cmd], shell=True)
|
4292
5289
|
|
4293
5290
|
else:
|
4294
|
-
return
|
4295
|
-
cmd=[tgt.python, '-c', src],
|
4296
|
-
shell=False,
|
4297
|
-
)
|
5291
|
+
return SubprocessRemoteSpawning._PreparedCmd([tgt.python, '-c', src], shell=False)
|
4298
5292
|
|
4299
5293
|
#
|
4300
5294
|
|
4301
|
-
@
|
4302
|
-
|
4303
|
-
stdin: ta.IO
|
4304
|
-
stdout: ta.IO
|
4305
|
-
stderr: ta.Optional[ta.IO]
|
4306
|
-
|
4307
|
-
@contextlib.contextmanager
|
4308
|
-
def spawn(
|
5295
|
+
@contextlib.asynccontextmanager
|
5296
|
+
async def spawn(
|
4309
5297
|
self,
|
4310
|
-
tgt: Target,
|
5298
|
+
tgt: RemoteSpawning.Target,
|
4311
5299
|
src: str,
|
4312
5300
|
*,
|
4313
5301
|
timeout: ta.Optional[float] = None,
|
4314
|
-
|
5302
|
+
debug: bool = False,
|
5303
|
+
) -> ta.AsyncGenerator[RemoteSpawning.Spawned, None]:
|
4315
5304
|
pc = self._prepare_cmd(tgt, src)
|
4316
5305
|
|
4317
|
-
|
4318
|
-
|
4319
|
-
|
4320
|
-
|
4321
|
-
|
4322
|
-
|
4323
|
-
|
4324
|
-
|
4325
|
-
|
5306
|
+
cmd = pc.cmd
|
5307
|
+
if not debug:
|
5308
|
+
cmd = subprocess_maybe_shell_wrap_exec(*cmd)
|
5309
|
+
|
5310
|
+
async with asyncio_subprocess_popen(
|
5311
|
+
*cmd,
|
5312
|
+
shell=pc.shell,
|
5313
|
+
stdin=subprocess.PIPE,
|
5314
|
+
stdout=subprocess.PIPE,
|
5315
|
+
stderr=(
|
5316
|
+
SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, tgt.stderr)]
|
5317
|
+
if tgt.stderr is not None else None
|
5318
|
+
),
|
5319
|
+
timeout=timeout,
|
4326
5320
|
) as proc:
|
4327
5321
|
stdin = check_not_none(proc.stdin)
|
4328
5322
|
stdout = check_not_none(proc.stdout)
|
@@ -4340,8 +5334,6 @@ class RemoteSpawning:
|
|
4340
5334
|
except BrokenPipeError:
|
4341
5335
|
pass
|
4342
5336
|
|
4343
|
-
proc.wait(timeout)
|
4344
|
-
|
4345
5337
|
|
4346
5338
|
########################################
|
4347
5339
|
# ../../../omdev/interp/providers.py
|
@@ -4370,17 +5362,17 @@ class InterpProvider(abc.ABC):
|
|
4370
5362
|
setattr(cls, 'name', snake_case(cls.__name__[:-len(sfx)]))
|
4371
5363
|
|
4372
5364
|
@abc.abstractmethod
|
4373
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5365
|
+
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Awaitable[ta.Sequence[InterpVersion]]:
|
4374
5366
|
raise NotImplementedError
|
4375
5367
|
|
4376
5368
|
@abc.abstractmethod
|
4377
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5369
|
+
def get_installed_version(self, version: InterpVersion) -> ta.Awaitable[Interp]:
|
4378
5370
|
raise NotImplementedError
|
4379
5371
|
|
4380
|
-
def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5372
|
+
async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
4381
5373
|
return []
|
4382
5374
|
|
4383
|
-
def install_version(self, version: InterpVersion) -> Interp:
|
5375
|
+
async def install_version(self, version: InterpVersion) -> Interp:
|
4384
5376
|
raise TypeError
|
4385
5377
|
|
4386
5378
|
|
@@ -4392,10 +5384,10 @@ class RunningInterpProvider(InterpProvider):
|
|
4392
5384
|
def version(self) -> InterpVersion:
|
4393
5385
|
return InterpInspector.running().iv
|
4394
5386
|
|
4395
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5387
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
4396
5388
|
return [self.version()]
|
4397
5389
|
|
4398
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5390
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
4399
5391
|
if version != self.version():
|
4400
5392
|
raise KeyError(version)
|
4401
5393
|
return Interp(
|
@@ -4405,159 +5397,26 @@ class RunningInterpProvider(InterpProvider):
|
|
4405
5397
|
|
4406
5398
|
|
4407
5399
|
########################################
|
4408
|
-
# ../remote/
|
4409
|
-
|
4410
|
-
|
4411
|
-
##
|
4412
|
-
|
4413
|
-
|
4414
|
-
class _RemoteExecutionLogHandler(logging.Handler):
|
4415
|
-
def __init__(self, fn: ta.Callable[[str], None]) -> None:
|
4416
|
-
super().__init__()
|
4417
|
-
self._fn = fn
|
4418
|
-
|
4419
|
-
def emit(self, record):
|
4420
|
-
msg = self.format(record)
|
4421
|
-
self._fn(msg)
|
4422
|
-
|
4423
|
-
|
4424
|
-
@dc.dataclass(frozen=True)
|
4425
|
-
class _RemoteExecutionRequest:
|
4426
|
-
c: Command
|
4427
|
-
|
4428
|
-
|
4429
|
-
@dc.dataclass(frozen=True)
|
4430
|
-
class _RemoteExecutionLog:
|
4431
|
-
s: str
|
4432
|
-
|
4433
|
-
|
4434
|
-
@dc.dataclass(frozen=True)
|
4435
|
-
class _RemoteExecutionResponse:
|
4436
|
-
r: ta.Optional[CommandOutputOrExceptionData] = None
|
4437
|
-
l: ta.Optional[_RemoteExecutionLog] = None
|
4438
|
-
|
4439
|
-
|
4440
|
-
def _remote_execution_main() -> None:
|
4441
|
-
rt = pyremote_bootstrap_finalize() # noqa
|
4442
|
-
|
4443
|
-
chan = RemoteChannel(
|
4444
|
-
rt.input,
|
4445
|
-
rt.output,
|
4446
|
-
)
|
4447
|
-
|
4448
|
-
bs = check_not_none(chan.recv_obj(MainBootstrap))
|
4449
|
-
|
4450
|
-
if (prd := bs.remote_config.pycharm_remote_debug) is not None:
|
4451
|
-
pycharm_debug_connect(prd)
|
4452
|
-
|
4453
|
-
injector = main_bootstrap(bs)
|
4454
|
-
|
4455
|
-
chan.set_marshaler(injector[ObjMarshalerManager])
|
4456
|
-
|
4457
|
-
#
|
4458
|
-
|
4459
|
-
log_lock = threading.RLock()
|
4460
|
-
send_logs = False
|
4461
|
-
|
4462
|
-
def log_fn(s: str) -> None:
|
4463
|
-
with log_lock:
|
4464
|
-
if send_logs:
|
4465
|
-
chan.send_obj(_RemoteExecutionResponse(l=_RemoteExecutionLog(s)))
|
4466
|
-
|
4467
|
-
log_handler = _RemoteExecutionLogHandler(log_fn)
|
4468
|
-
logging.root.addHandler(log_handler)
|
4469
|
-
|
4470
|
-
#
|
4471
|
-
|
4472
|
-
ce = injector[LocalCommandExecutor]
|
4473
|
-
|
4474
|
-
while True:
|
4475
|
-
req = chan.recv_obj(_RemoteExecutionRequest)
|
4476
|
-
if req is None:
|
4477
|
-
break
|
4478
|
-
|
4479
|
-
with log_lock:
|
4480
|
-
send_logs = True
|
4481
|
-
|
4482
|
-
r = ce.try_execute(
|
4483
|
-
req.c,
|
4484
|
-
log=log,
|
4485
|
-
omit_exc_object=True,
|
4486
|
-
)
|
4487
|
-
|
4488
|
-
with log_lock:
|
4489
|
-
send_logs = False
|
4490
|
-
|
4491
|
-
chan.send_obj(_RemoteExecutionResponse(r=CommandOutputOrExceptionData(
|
4492
|
-
output=r.output,
|
4493
|
-
exception=r.exception,
|
4494
|
-
)))
|
5400
|
+
# ../remote/connection.py
|
4495
5401
|
|
4496
5402
|
|
4497
5403
|
##
|
4498
5404
|
|
4499
5405
|
|
4500
|
-
|
4501
|
-
|
4502
|
-
|
4503
|
-
|
4504
|
-
|
4505
|
-
class RemoteCommandExecutor(CommandExecutor):
|
4506
|
-
def __init__(self, chan: RemoteChannel) -> None:
|
4507
|
-
super().__init__()
|
4508
|
-
|
4509
|
-
self._chan = chan
|
4510
|
-
|
4511
|
-
def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
|
4512
|
-
self._chan.send_obj(_RemoteExecutionRequest(cmd))
|
4513
|
-
|
4514
|
-
while True:
|
4515
|
-
if (r := self._chan.recv_obj(_RemoteExecutionResponse)) is None:
|
4516
|
-
raise EOFError
|
4517
|
-
|
4518
|
-
if r.l is not None:
|
4519
|
-
log.info(r.l.s)
|
4520
|
-
|
4521
|
-
if r.r is not None:
|
4522
|
-
return r.r
|
4523
|
-
|
4524
|
-
# @ta.override
|
4525
|
-
def execute(self, cmd: Command) -> Command.Output:
|
4526
|
-
r = self._remote_execute(cmd)
|
4527
|
-
if (e := r.exception) is not None:
|
4528
|
-
raise RemoteCommandError(e)
|
4529
|
-
else:
|
4530
|
-
return check_not_none(r.output)
|
4531
|
-
|
4532
|
-
# @ta.override
|
4533
|
-
def try_execute(
|
5406
|
+
class RemoteExecutionConnector(abc.ABC):
|
5407
|
+
@abc.abstractmethod
|
5408
|
+
def connect(
|
4534
5409
|
self,
|
4535
|
-
|
4536
|
-
|
4537
|
-
|
4538
|
-
|
4539
|
-
) -> CommandOutputOrException:
|
4540
|
-
try:
|
4541
|
-
r = self._remote_execute(cmd)
|
4542
|
-
|
4543
|
-
except Exception as e: # noqa
|
4544
|
-
if log is not None:
|
4545
|
-
log.exception('Exception executing remote command: %r', type(cmd))
|
4546
|
-
|
4547
|
-
return CommandOutputOrExceptionData(exception=CommandException.of(
|
4548
|
-
e,
|
4549
|
-
omit_exc_object=omit_exc_object,
|
4550
|
-
cmd=cmd,
|
4551
|
-
))
|
4552
|
-
|
4553
|
-
else:
|
4554
|
-
return r
|
5410
|
+
tgt: RemoteSpawning.Target,
|
5411
|
+
bs: MainBootstrap,
|
5412
|
+
) -> ta.AsyncContextManager[RemoteCommandExecutor]:
|
5413
|
+
raise NotImplementedError
|
4555
5414
|
|
4556
5415
|
|
4557
5416
|
##
|
4558
5417
|
|
4559
5418
|
|
4560
|
-
class
|
5419
|
+
class PyremoteRemoteExecutionConnector(RemoteExecutionConnector):
|
4561
5420
|
def __init__(
|
4562
5421
|
self,
|
4563
5422
|
*,
|
@@ -4590,38 +5449,43 @@ class RemoteExecution:
|
|
4590
5449
|
|
4591
5450
|
#
|
4592
5451
|
|
4593
|
-
@contextlib.
|
4594
|
-
def connect(
|
5452
|
+
@contextlib.asynccontextmanager
|
5453
|
+
async def connect(
|
4595
5454
|
self,
|
4596
5455
|
tgt: RemoteSpawning.Target,
|
4597
5456
|
bs: MainBootstrap,
|
4598
|
-
) -> ta.
|
5457
|
+
) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
|
4599
5458
|
spawn_src = self._spawn_src()
|
4600
5459
|
remote_src = self._remote_src()
|
4601
5460
|
|
4602
|
-
with self._spawning.spawn(
|
5461
|
+
async with self._spawning.spawn(
|
4603
5462
|
tgt,
|
4604
5463
|
spawn_src,
|
5464
|
+
debug=bs.main_config.debug,
|
4605
5465
|
) as proc:
|
4606
|
-
res = PyremoteBootstrapDriver( # noqa
|
5466
|
+
res = await PyremoteBootstrapDriver( # noqa
|
4607
5467
|
remote_src,
|
4608
5468
|
PyremoteBootstrapOptions(
|
4609
5469
|
debug=bs.main_config.debug,
|
4610
5470
|
),
|
4611
|
-
).
|
5471
|
+
).async_run(
|
4612
5472
|
proc.stdout,
|
4613
5473
|
proc.stdin,
|
4614
5474
|
)
|
4615
5475
|
|
4616
|
-
chan =
|
5476
|
+
chan = RemoteChannelImpl(
|
4617
5477
|
proc.stdout,
|
4618
5478
|
proc.stdin,
|
4619
5479
|
msh=self._msh,
|
4620
5480
|
)
|
4621
5481
|
|
4622
|
-
chan.send_obj(bs)
|
5482
|
+
await chan.send_obj(bs)
|
5483
|
+
|
5484
|
+
rce: RemoteCommandExecutor
|
5485
|
+
async with contextlib.aclosing(RemoteCommandExecutor(chan)) as rce:
|
5486
|
+
await rce.start()
|
4623
5487
|
|
4624
|
-
|
5488
|
+
yield rce
|
4625
5489
|
|
4626
5490
|
|
4627
5491
|
########################################
|
@@ -4643,7 +5507,6 @@ TODO:
|
|
4643
5507
|
|
4644
5508
|
|
4645
5509
|
class Pyenv:
|
4646
|
-
|
4647
5510
|
def __init__(
|
4648
5511
|
self,
|
4649
5512
|
*,
|
@@ -4656,13 +5519,13 @@ class Pyenv:
|
|
4656
5519
|
|
4657
5520
|
self._root_kw = root
|
4658
5521
|
|
4659
|
-
@
|
4660
|
-
def root(self) -> ta.Optional[str]:
|
5522
|
+
@async_cached_nullary
|
5523
|
+
async def root(self) -> ta.Optional[str]:
|
4661
5524
|
if self._root_kw is not None:
|
4662
5525
|
return self._root_kw
|
4663
5526
|
|
4664
5527
|
if shutil.which('pyenv'):
|
4665
|
-
return
|
5528
|
+
return await asyncio_subprocess_check_output_str('pyenv', 'root')
|
4666
5529
|
|
4667
5530
|
d = os.path.expanduser('~/.pyenv')
|
4668
5531
|
if os.path.isdir(d) and os.path.isfile(os.path.join(d, 'bin', 'pyenv')):
|
@@ -4670,12 +5533,12 @@ class Pyenv:
|
|
4670
5533
|
|
4671
5534
|
return None
|
4672
5535
|
|
4673
|
-
@
|
4674
|
-
def exe(self) -> str:
|
4675
|
-
return os.path.join(check_not_none(self.root()), 'bin', 'pyenv')
|
5536
|
+
@async_cached_nullary
|
5537
|
+
async def exe(self) -> str:
|
5538
|
+
return os.path.join(check_not_none(await self.root()), 'bin', 'pyenv')
|
4676
5539
|
|
4677
|
-
def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
|
4678
|
-
if (root := self.root()) is None:
|
5540
|
+
async def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
|
5541
|
+
if (root := await self.root()) is None:
|
4679
5542
|
return []
|
4680
5543
|
ret = []
|
4681
5544
|
vp = os.path.join(root, 'versions')
|
@@ -4687,11 +5550,11 @@ class Pyenv:
|
|
4687
5550
|
ret.append((dn, ep))
|
4688
5551
|
return ret
|
4689
5552
|
|
4690
|
-
def installable_versions(self) -> ta.List[str]:
|
4691
|
-
if self.root() is None:
|
5553
|
+
async def installable_versions(self) -> ta.List[str]:
|
5554
|
+
if await self.root() is None:
|
4692
5555
|
return []
|
4693
5556
|
ret = []
|
4694
|
-
s =
|
5557
|
+
s = await asyncio_subprocess_check_output_str(await self.exe(), 'install', '--list')
|
4695
5558
|
for l in s.splitlines():
|
4696
5559
|
if not l.startswith(' '):
|
4697
5560
|
continue
|
@@ -4701,12 +5564,12 @@ class Pyenv:
|
|
4701
5564
|
ret.append(l)
|
4702
5565
|
return ret
|
4703
5566
|
|
4704
|
-
def update(self) -> bool:
|
4705
|
-
if (root := self.root()) is None:
|
5567
|
+
async def update(self) -> bool:
|
5568
|
+
if (root := await self.root()) is None:
|
4706
5569
|
return False
|
4707
5570
|
if not os.path.isdir(os.path.join(root, '.git')):
|
4708
5571
|
return False
|
4709
|
-
|
5572
|
+
await asyncio_subprocess_check_call('git', 'pull', cwd=root)
|
4710
5573
|
return True
|
4711
5574
|
|
4712
5575
|
|
@@ -4767,17 +5630,16 @@ THREADED_PYENV_INSTALL_OPTS = PyenvInstallOpts(conf_opts=['--disable-gil'])
|
|
4767
5630
|
|
4768
5631
|
class PyenvInstallOptsProvider(abc.ABC):
|
4769
5632
|
@abc.abstractmethod
|
4770
|
-
def opts(self) -> PyenvInstallOpts:
|
5633
|
+
def opts(self) -> ta.Awaitable[PyenvInstallOpts]:
|
4771
5634
|
raise NotImplementedError
|
4772
5635
|
|
4773
5636
|
|
4774
5637
|
class LinuxPyenvInstallOpts(PyenvInstallOptsProvider):
|
4775
|
-
def opts(self) -> PyenvInstallOpts:
|
5638
|
+
async def opts(self) -> PyenvInstallOpts:
|
4776
5639
|
return PyenvInstallOpts()
|
4777
5640
|
|
4778
5641
|
|
4779
5642
|
class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
4780
|
-
|
4781
5643
|
@cached_nullary
|
4782
5644
|
def framework_opts(self) -> PyenvInstallOpts:
|
4783
5645
|
return PyenvInstallOpts(conf_opts=['--enable-framework'])
|
@@ -4793,12 +5655,12 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
4793
5655
|
'zlib',
|
4794
5656
|
]
|
4795
5657
|
|
4796
|
-
@
|
4797
|
-
def brew_deps_opts(self) -> PyenvInstallOpts:
|
5658
|
+
@async_cached_nullary
|
5659
|
+
async def brew_deps_opts(self) -> PyenvInstallOpts:
|
4798
5660
|
cflags = []
|
4799
5661
|
ldflags = []
|
4800
5662
|
for dep in self.BREW_DEPS:
|
4801
|
-
dep_prefix =
|
5663
|
+
dep_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', dep)
|
4802
5664
|
cflags.append(f'-I{dep_prefix}/include')
|
4803
5665
|
ldflags.append(f'-L{dep_prefix}/lib')
|
4804
5666
|
return PyenvInstallOpts(
|
@@ -4806,13 +5668,13 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
4806
5668
|
ldflags=ldflags,
|
4807
5669
|
)
|
4808
5670
|
|
4809
|
-
@
|
4810
|
-
def brew_tcl_opts(self) -> PyenvInstallOpts:
|
4811
|
-
if
|
5671
|
+
@async_cached_nullary
|
5672
|
+
async def brew_tcl_opts(self) -> PyenvInstallOpts:
|
5673
|
+
if await asyncio_subprocess_try_output('brew', '--prefix', 'tcl-tk') is None:
|
4812
5674
|
return PyenvInstallOpts()
|
4813
5675
|
|
4814
|
-
tcl_tk_prefix =
|
4815
|
-
tcl_tk_ver_str =
|
5676
|
+
tcl_tk_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', 'tcl-tk')
|
5677
|
+
tcl_tk_ver_str = await asyncio_subprocess_check_output_str('brew', 'ls', '--versions', 'tcl-tk')
|
4816
5678
|
tcl_tk_ver = '.'.join(tcl_tk_ver_str.split()[1].split('.')[:2])
|
4817
5679
|
|
4818
5680
|
return PyenvInstallOpts(conf_opts=[
|
@@ -4827,11 +5689,11 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
4827
5689
|
# pkg_config_path += ':' + os.environ['PKG_CONFIG_PATH']
|
4828
5690
|
# return PyenvInstallOpts(env={'PKG_CONFIG_PATH': pkg_config_path})
|
4829
5691
|
|
4830
|
-
def opts(self) -> PyenvInstallOpts:
|
5692
|
+
async def opts(self) -> PyenvInstallOpts:
|
4831
5693
|
return PyenvInstallOpts().merge(
|
4832
5694
|
self.framework_opts(),
|
4833
|
-
self.brew_deps_opts(),
|
4834
|
-
self.brew_tcl_opts(),
|
5695
|
+
await self.brew_deps_opts(),
|
5696
|
+
await self.brew_tcl_opts(),
|
4835
5697
|
# self.brew_ssl_opts(),
|
4836
5698
|
)
|
4837
5699
|
|
@@ -4863,20 +5725,8 @@ class PyenvVersionInstaller:
|
|
4863
5725
|
) -> None:
|
4864
5726
|
super().__init__()
|
4865
5727
|
|
4866
|
-
if no_default_opts:
|
4867
|
-
if opts is None:
|
4868
|
-
opts = PyenvInstallOpts()
|
4869
|
-
else:
|
4870
|
-
lst = [opts if opts is not None else DEFAULT_PYENV_INSTALL_OPTS]
|
4871
|
-
if interp_opts.debug:
|
4872
|
-
lst.append(DEBUG_PYENV_INSTALL_OPTS)
|
4873
|
-
if interp_opts.threaded:
|
4874
|
-
lst.append(THREADED_PYENV_INSTALL_OPTS)
|
4875
|
-
lst.append(PLATFORM_PYENV_INSTALL_OPTS[sys.platform].opts())
|
4876
|
-
opts = PyenvInstallOpts().merge(*lst)
|
4877
|
-
|
4878
5728
|
self._version = version
|
4879
|
-
self.
|
5729
|
+
self._given_opts = opts
|
4880
5730
|
self._interp_opts = interp_opts
|
4881
5731
|
self._given_install_name = install_name
|
4882
5732
|
|
@@ -4887,9 +5737,21 @@ class PyenvVersionInstaller:
|
|
4887
5737
|
def version(self) -> str:
|
4888
5738
|
return self._version
|
4889
5739
|
|
4890
|
-
@
|
4891
|
-
def opts(self) -> PyenvInstallOpts:
|
4892
|
-
|
5740
|
+
@async_cached_nullary
|
5741
|
+
async def opts(self) -> PyenvInstallOpts:
|
5742
|
+
opts = self._given_opts
|
5743
|
+
if self._no_default_opts:
|
5744
|
+
if opts is None:
|
5745
|
+
opts = PyenvInstallOpts()
|
5746
|
+
else:
|
5747
|
+
lst = [self._given_opts if self._given_opts is not None else DEFAULT_PYENV_INSTALL_OPTS]
|
5748
|
+
if self._interp_opts.debug:
|
5749
|
+
lst.append(DEBUG_PYENV_INSTALL_OPTS)
|
5750
|
+
if self._interp_opts.threaded:
|
5751
|
+
lst.append(THREADED_PYENV_INSTALL_OPTS)
|
5752
|
+
lst.append(await PLATFORM_PYENV_INSTALL_OPTS[sys.platform].opts())
|
5753
|
+
opts = PyenvInstallOpts().merge(*lst)
|
5754
|
+
return opts
|
4893
5755
|
|
4894
5756
|
@cached_nullary
|
4895
5757
|
def install_name(self) -> str:
|
@@ -4897,17 +5759,18 @@ class PyenvVersionInstaller:
|
|
4897
5759
|
return self._given_install_name
|
4898
5760
|
return self._version + ('-debug' if self._interp_opts.debug else '')
|
4899
5761
|
|
4900
|
-
@
|
4901
|
-
def install_dir(self) -> str:
|
4902
|
-
return str(os.path.join(check_not_none(self._pyenv.root()), 'versions', self.install_name()))
|
5762
|
+
@async_cached_nullary
|
5763
|
+
async def install_dir(self) -> str:
|
5764
|
+
return str(os.path.join(check_not_none(await self._pyenv.root()), 'versions', self.install_name()))
|
4903
5765
|
|
4904
|
-
@
|
4905
|
-
def install(self) -> str:
|
4906
|
-
|
5766
|
+
@async_cached_nullary
|
5767
|
+
async def install(self) -> str:
|
5768
|
+
opts = await self.opts()
|
5769
|
+
env = {**os.environ, **opts.env}
|
4907
5770
|
for k, l in [
|
4908
|
-
('CFLAGS',
|
4909
|
-
('LDFLAGS',
|
4910
|
-
('PYTHON_CONFIGURE_OPTS',
|
5771
|
+
('CFLAGS', opts.cflags),
|
5772
|
+
('LDFLAGS', opts.ldflags),
|
5773
|
+
('PYTHON_CONFIGURE_OPTS', opts.conf_opts),
|
4911
5774
|
]:
|
4912
5775
|
v = ' '.join(l)
|
4913
5776
|
if k in os.environ:
|
@@ -4915,13 +5778,13 @@ class PyenvVersionInstaller:
|
|
4915
5778
|
env[k] = v
|
4916
5779
|
|
4917
5780
|
conf_args = [
|
4918
|
-
*
|
5781
|
+
*opts.opts,
|
4919
5782
|
self._version,
|
4920
5783
|
]
|
4921
5784
|
|
4922
5785
|
if self._given_install_name is not None:
|
4923
5786
|
full_args = [
|
4924
|
-
os.path.join(check_not_none(self._pyenv.root()), 'plugins', 'python-build', 'bin', 'python-build'),
|
5787
|
+
os.path.join(check_not_none(await self._pyenv.root()), 'plugins', 'python-build', 'bin', 'python-build'), # noqa
|
4925
5788
|
*conf_args,
|
4926
5789
|
self.install_dir(),
|
4927
5790
|
]
|
@@ -4932,12 +5795,12 @@ class PyenvVersionInstaller:
|
|
4932
5795
|
*conf_args,
|
4933
5796
|
]
|
4934
5797
|
|
4935
|
-
|
5798
|
+
await asyncio_subprocess_check_call(
|
4936
5799
|
*full_args,
|
4937
5800
|
env=env,
|
4938
5801
|
)
|
4939
5802
|
|
4940
|
-
exe = os.path.join(self.install_dir(), 'bin', 'python')
|
5803
|
+
exe = os.path.join(await self.install_dir(), 'bin', 'python')
|
4941
5804
|
if not os.path.isfile(exe):
|
4942
5805
|
raise RuntimeError(f'Interpreter not found: {exe}')
|
4943
5806
|
return exe
|
@@ -4947,7 +5810,6 @@ class PyenvVersionInstaller:
|
|
4947
5810
|
|
4948
5811
|
|
4949
5812
|
class PyenvInterpProvider(InterpProvider):
|
4950
|
-
|
4951
5813
|
def __init__(
|
4952
5814
|
self,
|
4953
5815
|
pyenv: Pyenv = Pyenv(),
|
@@ -4990,11 +5852,11 @@ class PyenvInterpProvider(InterpProvider):
|
|
4990
5852
|
exe: str
|
4991
5853
|
version: InterpVersion
|
4992
5854
|
|
4993
|
-
def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
|
5855
|
+
async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
|
4994
5856
|
iv: ta.Optional[InterpVersion]
|
4995
5857
|
if self._inspect:
|
4996
5858
|
try:
|
4997
|
-
iv = check_not_none(self._inspector.inspect(ep)).iv
|
5859
|
+
iv = check_not_none(await self._inspector.inspect(ep)).iv
|
4998
5860
|
except Exception as e: # noqa
|
4999
5861
|
return None
|
5000
5862
|
else:
|
@@ -5007,10 +5869,10 @@ class PyenvInterpProvider(InterpProvider):
|
|
5007
5869
|
version=iv,
|
5008
5870
|
)
|
5009
5871
|
|
5010
|
-
def installed(self) -> ta.Sequence[Installed]:
|
5872
|
+
async def installed(self) -> ta.Sequence[Installed]:
|
5011
5873
|
ret: ta.List[PyenvInterpProvider.Installed] = []
|
5012
|
-
for vn, ep in self._pyenv.version_exes():
|
5013
|
-
if (i := self._make_installed(vn, ep)) is None:
|
5874
|
+
for vn, ep in await self._pyenv.version_exes():
|
5875
|
+
if (i := await self._make_installed(vn, ep)) is None:
|
5014
5876
|
log.debug('Invalid pyenv version: %s', vn)
|
5015
5877
|
continue
|
5016
5878
|
ret.append(i)
|
@@ -5018,11 +5880,11 @@ class PyenvInterpProvider(InterpProvider):
|
|
5018
5880
|
|
5019
5881
|
#
|
5020
5882
|
|
5021
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5022
|
-
return [i.version for i in self.installed()]
|
5883
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5884
|
+
return [i.version for i in await self.installed()]
|
5023
5885
|
|
5024
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5025
|
-
for i in self.installed():
|
5886
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
5887
|
+
for i in await self.installed():
|
5026
5888
|
if i.version == version:
|
5027
5889
|
return Interp(
|
5028
5890
|
exe=i.exe,
|
@@ -5032,10 +5894,10 @@ class PyenvInterpProvider(InterpProvider):
|
|
5032
5894
|
|
5033
5895
|
#
|
5034
5896
|
|
5035
|
-
def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5897
|
+
async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5036
5898
|
lst = []
|
5037
5899
|
|
5038
|
-
for vs in self._pyenv.installable_versions():
|
5900
|
+
for vs in await self._pyenv.installable_versions():
|
5039
5901
|
if (iv := self.guess_version(vs)) is None:
|
5040
5902
|
continue
|
5041
5903
|
if iv.opts.debug:
|
@@ -5045,16 +5907,16 @@ class PyenvInterpProvider(InterpProvider):
|
|
5045
5907
|
|
5046
5908
|
return lst
|
5047
5909
|
|
5048
|
-
def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5049
|
-
lst = self._get_installable_versions(spec)
|
5910
|
+
async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5911
|
+
lst = await self._get_installable_versions(spec)
|
5050
5912
|
|
5051
5913
|
if self._try_update and not any(v in spec for v in lst):
|
5052
5914
|
if self._pyenv.update():
|
5053
|
-
lst = self._get_installable_versions(spec)
|
5915
|
+
lst = await self._get_installable_versions(spec)
|
5054
5916
|
|
5055
5917
|
return lst
|
5056
5918
|
|
5057
|
-
def install_version(self, version: InterpVersion) -> Interp:
|
5919
|
+
async def install_version(self, version: InterpVersion) -> Interp:
|
5058
5920
|
inst_version = str(version.version)
|
5059
5921
|
inst_opts = version.opts
|
5060
5922
|
if inst_opts.threaded:
|
@@ -5066,7 +5928,7 @@ class PyenvInterpProvider(InterpProvider):
|
|
5066
5928
|
interp_opts=inst_opts,
|
5067
5929
|
)
|
5068
5930
|
|
5069
|
-
exe = installer.install()
|
5931
|
+
exe = await installer.install()
|
5070
5932
|
return Interp(exe, version)
|
5071
5933
|
|
5072
5934
|
|
@@ -5145,7 +6007,7 @@ class SystemInterpProvider(InterpProvider):
|
|
5145
6007
|
|
5146
6008
|
#
|
5147
6009
|
|
5148
|
-
def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
|
6010
|
+
async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
|
5149
6011
|
if not self.inspect:
|
5150
6012
|
s = os.path.basename(exe)
|
5151
6013
|
if s.startswith('python'):
|
@@ -5155,13 +6017,13 @@ class SystemInterpProvider(InterpProvider):
|
|
5155
6017
|
return InterpVersion.parse(s)
|
5156
6018
|
except InvalidVersion:
|
5157
6019
|
pass
|
5158
|
-
ii = self.inspector.inspect(exe)
|
6020
|
+
ii = await self.inspector.inspect(exe)
|
5159
6021
|
return ii.iv if ii is not None else None
|
5160
6022
|
|
5161
|
-
def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
|
6023
|
+
async def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
|
5162
6024
|
lst = []
|
5163
6025
|
for e in self.exes():
|
5164
|
-
if (ev := self.get_exe_version(e)) is None:
|
6026
|
+
if (ev := await self.get_exe_version(e)) is None:
|
5165
6027
|
log.debug('Invalid system version: %s', e)
|
5166
6028
|
continue
|
5167
6029
|
lst.append((e, ev))
|
@@ -5169,11 +6031,11 @@ class SystemInterpProvider(InterpProvider):
|
|
5169
6031
|
|
5170
6032
|
#
|
5171
6033
|
|
5172
|
-
def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
5173
|
-
return [ev for e, ev in self.exe_versions()]
|
6034
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
6035
|
+
return [ev for e, ev in await self.exe_versions()]
|
5174
6036
|
|
5175
|
-
def get_installed_version(self, version: InterpVersion) -> Interp:
|
5176
|
-
for e, ev in self.exe_versions():
|
6037
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
6038
|
+
for e, ev in await self.exe_versions():
|
5177
6039
|
if ev != version:
|
5178
6040
|
continue
|
5179
6041
|
return Interp(
|
@@ -5194,9 +6056,11 @@ def bind_remote(
|
|
5194
6056
|
lst: ta.List[InjectorBindingOrBindings] = [
|
5195
6057
|
inj.bind(remote_config),
|
5196
6058
|
|
5197
|
-
inj.bind(
|
6059
|
+
inj.bind(SubprocessRemoteSpawning, singleton=True),
|
6060
|
+
inj.bind(RemoteSpawning, to_key=SubprocessRemoteSpawning),
|
5198
6061
|
|
5199
|
-
inj.bind(
|
6062
|
+
inj.bind(PyremoteRemoteExecutionConnector, singleton=True),
|
6063
|
+
inj.bind(RemoteExecutionConnector, to_key=PyremoteRemoteExecutionConnector),
|
5200
6064
|
]
|
5201
6065
|
|
5202
6066
|
if (pf := remote_config.payload_file) is not None:
|
@@ -5220,13 +6084,14 @@ class InterpResolver:
|
|
5220
6084
|
providers: ta.Sequence[ta.Tuple[str, InterpProvider]],
|
5221
6085
|
) -> None:
|
5222
6086
|
super().__init__()
|
6087
|
+
|
5223
6088
|
self._providers: ta.Mapping[str, InterpProvider] = collections.OrderedDict(providers)
|
5224
6089
|
|
5225
|
-
def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
|
6090
|
+
async def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
|
5226
6091
|
lst = [
|
5227
6092
|
(i, si)
|
5228
6093
|
for i, p in enumerate(self._providers.values())
|
5229
|
-
for si in p.get_installed_versions(spec)
|
6094
|
+
for si in await p.get_installed_versions(spec)
|
5230
6095
|
if spec.contains(si)
|
5231
6096
|
]
|
5232
6097
|
|
@@ -5238,16 +6103,16 @@ class InterpResolver:
|
|
5238
6103
|
bp = list(self._providers.values())[bi]
|
5239
6104
|
return (bp, bv)
|
5240
6105
|
|
5241
|
-
def resolve(
|
6106
|
+
async def resolve(
|
5242
6107
|
self,
|
5243
6108
|
spec: InterpSpecifier,
|
5244
6109
|
*,
|
5245
6110
|
install: bool = False,
|
5246
6111
|
) -> ta.Optional[Interp]:
|
5247
|
-
tup = self._resolve_installed(spec)
|
6112
|
+
tup = await self._resolve_installed(spec)
|
5248
6113
|
if tup is not None:
|
5249
6114
|
bp, bv = tup
|
5250
|
-
return bp.get_installed_version(bv)
|
6115
|
+
return await bp.get_installed_version(bv)
|
5251
6116
|
|
5252
6117
|
if not install:
|
5253
6118
|
return None
|
@@ -5255,21 +6120,21 @@ class InterpResolver:
|
|
5255
6120
|
tp = list(self._providers.values())[0] # noqa
|
5256
6121
|
|
5257
6122
|
sv = sorted(
|
5258
|
-
[s for s in tp.get_installable_versions(spec) if s in spec],
|
6123
|
+
[s for s in await tp.get_installable_versions(spec) if s in spec],
|
5259
6124
|
key=lambda s: s.version,
|
5260
6125
|
)
|
5261
6126
|
if not sv:
|
5262
6127
|
return None
|
5263
6128
|
|
5264
6129
|
bv = sv[-1]
|
5265
|
-
return tp.install_version(bv)
|
6130
|
+
return await tp.install_version(bv)
|
5266
6131
|
|
5267
|
-
def list(self, spec: InterpSpecifier) -> None:
|
6132
|
+
async def list(self, spec: InterpSpecifier) -> None:
|
5268
6133
|
print('installed:')
|
5269
6134
|
for n, p in self._providers.items():
|
5270
6135
|
lst = [
|
5271
6136
|
si
|
5272
|
-
for si in p.get_installed_versions(spec)
|
6137
|
+
for si in await p.get_installed_versions(spec)
|
5273
6138
|
if spec.contains(si)
|
5274
6139
|
]
|
5275
6140
|
if lst:
|
@@ -5283,7 +6148,7 @@ class InterpResolver:
|
|
5283
6148
|
for n, p in self._providers.items():
|
5284
6149
|
lst = [
|
5285
6150
|
si
|
5286
|
-
for si in p.get_installable_versions(spec)
|
6151
|
+
for si in await p.get_installable_versions(spec)
|
5287
6152
|
if spec.contains(si)
|
5288
6153
|
]
|
5289
6154
|
if lst:
|
@@ -5325,9 +6190,9 @@ class InterpCommand(Command['InterpCommand.Output']):
|
|
5325
6190
|
|
5326
6191
|
|
5327
6192
|
class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
|
5328
|
-
def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
|
6193
|
+
async def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
|
5329
6194
|
i = InterpSpecifier.parse(check_not_none(cmd.spec))
|
5330
|
-
o = check_not_none(DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
|
6195
|
+
o = check_not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
|
5331
6196
|
return InterpCommand.Output(
|
5332
6197
|
exe=o.exe,
|
5333
6198
|
version=str(o.version.version),
|
@@ -5366,7 +6231,7 @@ def bind_command(
|
|
5366
6231
|
class _FactoryCommandExecutor(CommandExecutor):
|
5367
6232
|
factory: ta.Callable[[], CommandExecutor]
|
5368
6233
|
|
5369
|
-
def execute(self, i: Command) -> Command.Output:
|
6234
|
+
def execute(self, i: Command) -> ta.Awaitable[Command.Output]:
|
5370
6235
|
return self.factory().execute(i)
|
5371
6236
|
|
5372
6237
|
|
@@ -5521,31 +6386,7 @@ def main_bootstrap(bs: MainBootstrap) -> Injector:
|
|
5521
6386
|
##
|
5522
6387
|
|
5523
6388
|
|
5524
|
-
def
|
5525
|
-
import argparse
|
5526
|
-
|
5527
|
-
parser = argparse.ArgumentParser()
|
5528
|
-
|
5529
|
-
parser.add_argument('--_payload-file')
|
5530
|
-
|
5531
|
-
parser.add_argument('-s', '--shell')
|
5532
|
-
parser.add_argument('-q', '--shell-quote', action='store_true')
|
5533
|
-
parser.add_argument('--python', default='python3')
|
5534
|
-
|
5535
|
-
parser.add_argument('--pycharm-debug-port', type=int)
|
5536
|
-
parser.add_argument('--pycharm-debug-host')
|
5537
|
-
parser.add_argument('--pycharm-debug-version')
|
5538
|
-
|
5539
|
-
parser.add_argument('--debug', action='store_true')
|
5540
|
-
|
5541
|
-
parser.add_argument('--local', action='store_true')
|
5542
|
-
|
5543
|
-
parser.add_argument('command', nargs='+')
|
5544
|
-
|
5545
|
-
args = parser.parse_args()
|
5546
|
-
|
5547
|
-
#
|
5548
|
-
|
6389
|
+
async def _async_main(args: ta.Any) -> None:
|
5549
6390
|
bs = MainBootstrap(
|
5550
6391
|
main_config=MainConfig(
|
5551
6392
|
log_level='DEBUG' if args.debug else 'INFO',
|
@@ -5558,12 +6399,16 @@ def _main() -> None:
|
|
5558
6399
|
|
5559
6400
|
pycharm_remote_debug=PycharmRemoteDebug(
|
5560
6401
|
port=args.pycharm_debug_port,
|
5561
|
-
host=args.pycharm_debug_host,
|
6402
|
+
**(dict(host=args.pycharm_debug_host) if args.pycharm_debug_host is not None else {}),
|
5562
6403
|
install_version=args.pycharm_debug_version,
|
5563
6404
|
) if args.pycharm_debug_port is not None else None,
|
6405
|
+
|
6406
|
+
timebomb_delay_s=args.remote_timebomb_delay_s,
|
5564
6407
|
),
|
5565
6408
|
)
|
5566
6409
|
|
6410
|
+
#
|
6411
|
+
|
5567
6412
|
injector = main_bootstrap(
|
5568
6413
|
bs,
|
5569
6414
|
)
|
@@ -5582,7 +6427,7 @@ def _main() -> None:
|
|
5582
6427
|
|
5583
6428
|
#
|
5584
6429
|
|
5585
|
-
with contextlib.
|
6430
|
+
async with contextlib.AsyncExitStack() as es:
|
5586
6431
|
ce: CommandExecutor
|
5587
6432
|
|
5588
6433
|
if args.local:
|
@@ -5595,16 +6440,51 @@ def _main() -> None:
|
|
5595
6440
|
python=args.python,
|
5596
6441
|
)
|
5597
6442
|
|
5598
|
-
ce = es.
|
6443
|
+
ce = await es.enter_async_context(injector[RemoteExecutionConnector].connect(tgt, bs)) # noqa
|
5599
6444
|
|
5600
|
-
|
5601
|
-
|
6445
|
+
async def run_command(cmd: Command) -> None:
|
6446
|
+
res = await ce.try_execute(
|
5602
6447
|
cmd,
|
5603
6448
|
log=log,
|
5604
6449
|
omit_exc_object=True,
|
5605
6450
|
)
|
5606
6451
|
|
5607
|
-
print(msh.marshal_obj(
|
6452
|
+
print(msh.marshal_obj(res, opts=ObjMarshalOptions(raw_bytes=True)))
|
6453
|
+
|
6454
|
+
await asyncio.gather(*[
|
6455
|
+
run_command(cmd)
|
6456
|
+
for cmd in cmds
|
6457
|
+
])
|
6458
|
+
|
6459
|
+
|
6460
|
+
def _main() -> None:
|
6461
|
+
import argparse
|
6462
|
+
|
6463
|
+
parser = argparse.ArgumentParser()
|
6464
|
+
|
6465
|
+
parser.add_argument('--_payload-file')
|
6466
|
+
|
6467
|
+
parser.add_argument('-s', '--shell')
|
6468
|
+
parser.add_argument('-q', '--shell-quote', action='store_true')
|
6469
|
+
parser.add_argument('--python', default='python3')
|
6470
|
+
|
6471
|
+
parser.add_argument('--pycharm-debug-port', type=int)
|
6472
|
+
parser.add_argument('--pycharm-debug-host')
|
6473
|
+
parser.add_argument('--pycharm-debug-version')
|
6474
|
+
|
6475
|
+
parser.add_argument('--remote-timebomb-delay-s', type=float)
|
6476
|
+
|
6477
|
+
parser.add_argument('--debug', action='store_true')
|
6478
|
+
|
6479
|
+
parser.add_argument('--local', action='store_true')
|
6480
|
+
|
6481
|
+
parser.add_argument('command', nargs='+')
|
6482
|
+
|
6483
|
+
args = parser.parse_args()
|
6484
|
+
|
6485
|
+
#
|
6486
|
+
|
6487
|
+
asyncio.run(_async_main(args))
|
5608
6488
|
|
5609
6489
|
|
5610
6490
|
if __name__ == '__main__':
|